i18n-tasks 0.9.37 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9cad835aebcfc5621cb07d3a5ef4134a01cfed6286411776a9a5e04cfefe5862
-  data.tar.gz: 3b9f5a35077463778d1b940bd01738e49225fe7775d02858da5bdd9a9ded8308
+  metadata.gz: 66781586bda6c38de2ca767c265c7e1fbaf6e11184997f00a77c215e2270a7d7
+  data.tar.gz: 3a94d73244c34887758d208cdc807a930467ac6cde95545dea6acdbed5563c36
 SHA512:
-  metadata.gz: 8ed7363e9cfd2e8cf179aa0f28c0f00b926b4a53ba95f19ca61d8a08901322c561e84ebab51616f592400dac985f32c82b0a08911ec5f22223fe22e7858dd6af
-  data.tar.gz: 04b44a9180dc9b1e9d739eb5e5881877f68321bbd7efa1028a2f2d89fd3acf1cf742d587aab566a8cbd43813619693132ec803221d6c7ad31cd703ba516622f3
+  metadata.gz: b51f3bf32fc208a9845b79f9a074ed2bcd2f0ff648309b93dbf7431df7154103a63c298bdb197cfae611012423d315df1552ad600a20135dfd76592df5d853d8
+  data.tar.gz: 4cc284d8c283c4159333310f4deb2e78ec064f60c024ccdadd4007d883955ca08d1e765b69529744dff2f953973d3cab6ff46138919c5cc41c3c4f986ea2a88a

data/README.md

@@ -1,5 +1,7 @@
 # i18n-tasks [![Build Status][badge-ci]][ci] [![Coverage Status][badge-coverage]][coverage] [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/glebm/i18n-tasks?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
+[![Stand With Ukraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner2-direct.svg)](https://stand-with-ukraine.pp.ua/)
+
 i18n-tasks helps you find and manage missing and unused translations.
 
 <img width="539" height="331" src="https://i.imgur.com/XZBd8l7.png">
@@ -22,7 +24,7 @@ i18n-tasks can be used with any project using the ruby [i18n gem][i18n-gem] (def
 Add i18n-tasks to the Gemfile:
 
 ```ruby
-gem 'i18n-tasks', '~> 0.9.37'
+gem 'i18n-tasks', '~> 1.0.2'
 ```
 
 Copy the default [configuration file](#configuration):

data/i18n-tasks.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |s| # rubocop:disable Metrics/BlockLength
 
   s.add_dependency 'activesupport', '>= 4.0.2'
   s.add_dependency 'ast', '>= 2.1.0'
+  s.add_dependency 'better_html', '~> 1.0'
   s.add_dependency 'erubi'
   s.add_dependency 'highline', '>= 2.0.0'
   s.add_dependency 'i18n'

data/lib/i18n/tasks/command/commander.rb

@@ -16,6 +16,7 @@ module I18n::Tasks
       end
 
       def run(name, opts = {})
+        log_stderr "#{Rainbow('#StandWith').bg(:blue)}#{Rainbow('Ukraine').bg(:yellow)}"
         name = name.to_sym
         public_name = name.to_s.tr '_', '-'
         log_verbose "task: #{public_name}(#{opts.map { |k, v| "#{k}: #{v.inspect}" } * ', '})"

data/lib/i18n/tasks/data/adapter/yaml_adapter.rb

@@ -5,11 +5,13 @@ module I18n::Tasks
   module Data
     module Adapter
       module YamlAdapter
+        EMOJI_REGEX = /\\u[\da-f]{8}/i
+
         class << self
           # @return [Hash] locale tree
           def parse(str, options)
             if YAML.method(:load).arity.abs == 2
-              YAML.load(str, **(options || {}))
+              YAML.safe_load(str, **(options || {}), permitted_classes: [Symbol], aliases: true)
             else
               # older jruby and rbx 2.2.7 do not accept options
               YAML.load(str)
@@ -18,7 +20,12 @@ module I18n::Tasks
 
           # @return [String]
           def dump(tree, options)
-            tree.to_yaml(options || {})
+            restore_emojis(tree.to_yaml(options || {}))
+          end
+
+          # @return [String]
+          def restore_emojis(yaml)
+            yaml.gsub(EMOJI_REGEX) { |m| [m[-8..].to_i(16)].pack("U") }
           end
         end
       end

data/lib/i18n/tasks/scanners/erb_ast_processor.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'ast'
+require 'set'
+require 'i18n/tasks/scanners/local_ruby_parser'
+
+module I18n::Tasks::Scanners
+  class ErbAstProcessor
+    include AST::Processor::Mixin
+    def initialize
+      super()
+      @ruby_parser = LocalRubyParser.new(ignore_blocks: true)
+      @comments = []
+    end
+
+    def process_and_extract_comments(ast)
+      result = process(ast)
+      [result, @comments]
+    end
+
+    def on_code(node)
+      parsed, comments = @ruby_parser.parse(
+        node.children[0],
+        location: node.location
+      )
+      @comments.concat(comments)
+
+      unless parsed.nil?
+        parsed = parsed.updated(
+          nil,
+          parsed.children.map { |child| node?(child) ? process(child) : child }
+        )
+        node = node.updated(:send, parsed)
+      end
+      node
+    end
+
+    def handler_missing(node)
+      node.updated(
+        nil,
+        node.children.map { |child| node?(child) ? process(child) : child }
+      )
+    end
+
+    private
+
+    def node?(node)
+      node.is_a?(::Parser::AST::Node)
+    end
+  end
+end

data/lib/i18n/tasks/scanners/erb_ast_scanner.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks/scanners/ruby_ast_scanner'
+require 'i18n/tasks/scanners/erb_ast_processor'
+require 'better_html/errors'
+require 'better_html/parser'
+
+module I18n::Tasks::Scanners
+  # Scan for I18n.translate calls in ERB-file better-html and ASTs
+  class ErbAstScanner < RubyAstScanner
+    def initialize(**args)
+      super(**args)
+      @erb_ast_processor = ErbAstProcessor.new
+    end
+
+    private
+
+    # Parse file on path and returns AST and comments.
+    #
+    # @param path Path to file to parse
+    # @return [{Parser::AST::Node}, [Parser::Source::Comment]]
+    def path_to_ast_and_comments(path)
+      parser = BetterHtml::Parser.new(make_buffer(path))
+      ast = convert_better_html(parser.ast)
+      @erb_ast_processor.process_and_extract_comments(ast)
+    end
+
+    # Convert BetterHtml nodes to Parser::AST::Node
+    #
+    # @param node BetterHtml::Parser::AST::Node
+    # @return Parser::AST::Node
+    def convert_better_html(node)
+      definition = Parser::Source::Map::Definition.new(
+        node.location.begin,
+        node.location.begin,
+        node.location.begin,
+        node.location.end
+      )
+      Parser::AST::Node.new(
+        node.type,
+        node.children.map { |child| child.is_a?(BetterHtml::AST::Node) ? convert_better_html(child) : child },
+        {
+          location: definition
+        }
+      )
+    end
+  end
+end

data/lib/i18n/tasks/scanners/local_ruby_parser.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'parser/current'
+
+module I18n::Tasks::Scanners
+  class LocalRubyParser
+    # ignore_blocks feature inspired by shopify/better-html
+    # https://github.com/Shopify/better-html/blob/087943ffd2a5877fa977d71532010b0c91239519/lib/better_html/test_helper/ruby_node.rb#L24
+    BLOCK_EXPR = /\s*((\s+|\))do|\{)(\s*\|[^|]*\|)?\s*\Z/.freeze
+
+    def initialize(ignore_blocks: false)
+      @parser = ::Parser::CurrentRuby.new
+      @ignore_blocks = ignore_blocks
+    end
+
+    # Parse string and normalize location
+    def parse(source, location: nil)
+      buffer = ::Parser::Source::Buffer.new('(string)')
+      buffer.source = if @ignore_blocks
+                        source.sub(BLOCK_EXPR, '')
+                      else
+                        source
+                      end
+
+      @parser.reset
+      ast, comments = @parser.parse_with_comments(buffer)
+      ast = normalize_location(ast, location)
+      comments = comments.map { |comment| normalize_comment_location(comment, location) }
+      [ast, comments]
+    end
+
+    # Normalize location for all parsed nodes
+
+    # @param node {Parser::AST::Node} Node in parsed code
+    # @param location {Parser::Source::Map} Global location for the parsed string
+    # @return {Parser::AST::Node}
+    def normalize_location(node, location)
+      return node.map { |child| normalize_location(child, location) } if node.is_a?(Array)
+
+      return node unless node.is_a?(::Parser::AST::Node)
+
+      node.updated(
+        nil,
+        node.children.map { |child| normalize_location(child, location) },
+        { location: updated_location(location, node.location) }
+      )
+    end
+
+    # Calculate location relative to a global location
+    #
+    # @param global_location {Parser::Source::Map} Global location where the code was parsed
+    # @param local_location {Parser::Source::Map} Local location in the parsed string
+    # @return {Parser::Source::Map}
+    def updated_location(global_location, local_location)
+      range = ::Parser::Source::Range.new(
+        global_location.expression.source_buffer,
+        global_location.expression.to_range.begin + local_location.expression.to_range.begin,
+        global_location.expression.to_range.begin + local_location.expression.to_range.end
+      )
+
+      ::Parser::Source::Map::Definition.new(
+        range.begin,
+        range.begin,
+        range.begin,
+        range.end
+      )
+    end
+
+    # Normalize location for comment
+    #
+    # @param comment {Parser::Source::Comment} A comment with local location
+    # @param location {Parser::Source::Map} Global location for the parsed string
+    # @return {Parser::Source::Comment}
+    def normalize_comment_location(comment, location)
+      range = ::Parser::Source::Range.new(
+        location.expression.source_buffer,
+        location.expression.to_range.begin + comment.location.expression.to_range.begin,
+        location.expression.to_range.begin + comment.location.expression.to_range.end
+      )
+      ::Parser::Source::Comment.new(range)
+    end
+  end
+end

data/lib/i18n/tasks/scanners/results/occurrence.rb

@@ -48,7 +48,7 @@ module I18n::Tasks
         # rubocop:enable Metrics/ParameterLists
 
         def inspect
-          "Occurrence(#{@path}:#{@line_num}:#{@line_pos}:#{@pos}:#{@raw_key}:#{@default_arg})"
+          "Occurrence(path: #{@path}, line_num: #{@line_num}, line_pos: #{@line_pos}, pos: #{@pos}, raw_key: #{@raw_key}, default_arg: #{@default_arg}, line: #{@line})"
         end
 
         def ==(other)

data/lib/i18n/tasks/scanners/ruby_ast_call_finder.rb

@@ -2,6 +2,7 @@
 
 require 'ast'
 require 'set'
+
 module I18n::Tasks::Scanners
   class RubyAstCallFinder
     include AST::Processor::Mixin

data/lib/i18n/tasks/scanners/ruby_ast_scanner.rb

@@ -32,32 +32,22 @@ module I18n::Tasks::Scanners
     #
     # @return [Array<[key, Results::KeyOccurrence]>] each occurrence found in the file
     def scan_file(path)
-      @parser.reset
-      ast, comments = @parser.parse_with_comments(make_buffer(path))
-
-      results = @call_finder.collect_calls ast do |send_node, method_name|
-        send_node_to_key_occurrence(send_node, method_name)
-      end
+      ast, comments = path_to_ast_and_comments(path)
 
-      magic_comments  = comments.select { |comment| comment.text =~ MAGIC_COMMENT_PREFIX }
-      comment_to_node = Parser::Source::Comment.associate_locations(ast, magic_comments).tap do |h|
-        # transform_values is only available in ActiveSupport 4.2+
-        h.each { |k, v| h[k] = v.first }
-      end.invert
-      results + (magic_comments.flat_map do |comment|
-        @parser.reset
-        associated_node = comment_to_node[comment]
-        @call_finder.collect_calls(
-          @parser.parse(make_buffer(path, comment.text.sub(MAGIC_COMMENT_PREFIX, '').split(/\s+(?=t)/).join('; ')))
-        ) do |send_node, _method_name|
-          # method_name is not available at this stage
-          send_node_to_key_occurrence(send_node, nil, location: associated_node || comment.location)
-        end
-      end)
+      ast_to_occurences(ast) + comments_to_occurences(path, ast, comments)
     rescue Exception => e # rubocop:disable Lint/RescueException
       raise ::I18n::Tasks::CommandError.new(e, "Error scanning #{path}: #{e.message}")
     end
 
+    # Parse file on path and returns AST and comments.
+    #
+    # @param path Path to file to parse
+    # @return [{Parser::AST::Node}, [Parser::Source::Comment]]
+    def path_to_ast_and_comments(path)
+      @parser.reset
+      @parser.parse_with_comments(make_buffer(path))
+    end
+
     # @param send_node [Parser::AST::Node]
     # @param method_name [Symbol, nil]
     # @param location [Parser::Source::Map]
@@ -204,6 +194,41 @@ module I18n::Tasks::Scanners
         buffer.raw_source = contents
       end
     end
+
+    # Convert an array of {Parser::Source::Comment} to occurrences.
+    #
+    # @param path Path to file
+    # @param ast Parser::AST::Node
+    # @param comments [Parser::Source::Comment]
+    # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
+    def comments_to_occurences(path, ast, comments)
+      magic_comments = comments.select { |comment| comment.text =~ MAGIC_COMMENT_PREFIX }
+      comment_to_node = Parser::Source::Comment.associate_locations(ast, magic_comments).tap do |h|
+        # transform_values is only available in ActiveSupport 4.2+
+        h.each { |k, v| h[k] = v.first }
+      end.invert
+
+      magic_comments.flat_map do |comment|
+        @parser.reset
+        associated_node = comment_to_node[comment]
+        @call_finder.collect_calls(
+          @parser.parse(make_buffer(path, comment.text.sub(MAGIC_COMMENT_PREFIX, '').split(/\s+(?=t)/).join('; ')))
+        ) do |send_node, _method_name|
+          # method_name is not available at this stage
+          send_node_to_key_occurrence(send_node, nil, location: associated_node || comment.location)
+        end
+      end
+    end
+
+    # Convert {Parser::AST::Node} to occurrences.
+    #
+    # @param ast {Parser::Source::Comment}
+    # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
+    def ast_to_occurences(ast)
+      @call_finder.collect_calls(ast) do |send_node, method_name|
+        send_node_to_key_occurrence(send_node, method_name)
+      end
+    end
   end
 end
 # rubocop:enable Metrics/AbcSize,Metrics/BlockNesting,Metrics/PerceivedComplexity

data/lib/i18n/tasks/used_keys.rb

@@ -3,6 +3,7 @@
 require 'find'
 require 'i18n/tasks/scanners/pattern_with_scope_scanner'
 require 'i18n/tasks/scanners/ruby_ast_scanner'
+require 'i18n/tasks/scanners/erb_ast_scanner'
 require 'i18n/tasks/scanners/scanner_multiplexer'
 require 'i18n/tasks/scanners/files/caching_file_finder_provider'
 require 'i18n/tasks/scanners/files/caching_file_reader'
@@ -20,7 +21,8 @@ module I18n::Tasks
       relative_roots: %w[app/controllers app/helpers app/mailers app/presenters app/views].freeze,
       scanners: [
         ['::I18n::Tasks::Scanners::RubyAstScanner', { only: %w[*.rb] }],
-        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.rb] }]
+        ['::I18n::Tasks::Scanners::ErbAstScanner', { only: %w[*.erb] }],
+        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.erb *.rb] }]
       ],
       strict: true
     }.freeze

data/lib/i18n/tasks/version.rb

@@ -2,6 +2,6 @@
 
 module I18n
   module Tasks
-    VERSION = '0.9.37'
+    VERSION = '1.0.2'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: i18n-tasks
 version: !ruby/object:Gem::Version
-  version: 0.9.37
+  version: 1.0.2
 platform: ruby
 authors:
 - glebm
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-26 00:00:00.000000000 Z
+date: 2022-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: better_html
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: erubi
   requirement: !ruby/object:Gem::Requirement
@@ -357,12 +371,15 @@ files:
 - lib/i18n/tasks/references.rb
 - lib/i18n/tasks/reports/base.rb
 - lib/i18n/tasks/reports/terminal.rb
+- lib/i18n/tasks/scanners/erb_ast_processor.rb
+- lib/i18n/tasks/scanners/erb_ast_scanner.rb
 - lib/i18n/tasks/scanners/file_scanner.rb
 - lib/i18n/tasks/scanners/files/caching_file_finder.rb
 - lib/i18n/tasks/scanners/files/caching_file_finder_provider.rb
 - lib/i18n/tasks/scanners/files/caching_file_reader.rb
 - lib/i18n/tasks/scanners/files/file_finder.rb
 - lib/i18n/tasks/scanners/files/file_reader.rb
+- lib/i18n/tasks/scanners/local_ruby_parser.rb
 - lib/i18n/tasks/scanners/occurrence_from_position.rb
 - lib/i18n/tasks/scanners/pattern_mapper.rb
 - lib/i18n/tasks/scanners/pattern_scanner.rb