i18n-tasks 0.9.36 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cde1b3151cfb69975e8512b714de8707a8ad79a7dd38ec299a282450297399f9
-  data.tar.gz: f98141b89b24a49b163823af6351476024b1a676d0b89a781c49dc3a10ffc1a7
+  metadata.gz: 053f1f934ed84abbf4f86a11762c37c78cc286f8f809d3c8c5ca2918af57cb2a
+  data.tar.gz: 0f70179fa394de1faafc750185774d9a47f359de6b41c8542e038e8189933186
 SHA512:
-  metadata.gz: 364b4d1d23802bfbabab3c41c7c39677f263925459f6be39c1e746e1f07221bfb487b1dbc425e8643ebacaaa20b35c3f08eb5d00fabe7237b35556a30759a67b
-  data.tar.gz: 9c7889833088afd77d439cc3517b99f01100cb507ca329ce30b36193054c5747b98e7fae2b972d006daa6c9973376da02c8c137ff26471fe758409677d7cdc02
+  metadata.gz: 45934bf36576b5dfdf8e88f019779603ddcfe989c4d6ce5c3da75d07571b5097b1f06a7147663fadaef4995532d3d2661fc1733e2bf49396820202c14d33c005
+  data.tar.gz: 4853657f6fb46102d63d4a6f44ca0b07e3915ab7029369c815ec7530654973af30aee1184d826cc2df1a4a8c393525a3afa05005ef30aaabcf14435a66f693e6

data/README.md

@@ -1,5 +1,7 @@
 # i18n-tasks [![Build Status][badge-ci]][ci] [![Coverage Status][badge-coverage]][coverage] [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/glebm/i18n-tasks?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
+[![Stand With Ukraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner2-direct.svg)](https://stand-with-ukraine.pp.ua/)
+
 i18n-tasks helps you find and manage missing and unused translations.
 
 <img width="539" height="331" src="https://i.imgur.com/XZBd8l7.png">
@@ -22,7 +24,7 @@ i18n-tasks can be used with any project using the ruby [i18n gem][i18n-gem] (def
 Add i18n-tasks to the Gemfile:
 
 ```ruby
-gem 'i18n-tasks', '~> 0.9.36'
+gem 'i18n-tasks', '~> 1.0.1'
 ```
 
 Copy the default [configuration file](#configuration):

data/i18n-tasks.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |s| # rubocop:disable Metrics/BlockLength
   TEXT
   s.homepage = 'https://github.com/glebm/i18n-tasks'
   s.metadata = { 'issue_tracker' => 'https://github.com/glebm/i18n-tasks' } if s.respond_to?(:metadata=)
-  s.required_ruby_version = '>= 2.5', '< 4.0' if s.respond_to?(:required_ruby_version=)
+  s.required_ruby_version = '>= 2.6', '< 4.0' if s.respond_to?(:required_ruby_version=)
 
   s.files = `git ls-files`.split($/)
   s.files -= s.files.grep(%r{^(doc/|\.|spec/)}) + %w[CHANGES.md config/i18n-tasks.yml Gemfile]
@@ -35,6 +35,7 @@ Gem::Specification.new do |s| # rubocop:disable Metrics/BlockLength
 
   s.add_dependency 'activesupport', '>= 4.0.2'
   s.add_dependency 'ast', '>= 2.1.0'
+  s.add_dependency 'better_html', '~> 1.0'
   s.add_dependency 'erubi'
   s.add_dependency 'highline', '>= 2.0.0'
   s.add_dependency 'i18n'

data/lib/i18n/tasks/command/commander.rb

@@ -16,6 +16,7 @@ module I18n::Tasks
       end
 
       def run(name, opts = {})
+        log_stderr "#{Rainbow('#StandWith').bg(:blue)}#{Rainbow('Ukraine').bg(:yellow)}"
         name = name.to_sym
         public_name = name.to_s.tr '_', '-'
         log_verbose "task: #{public_name}(#{opts.map { |k, v| "#{k}: #{v.inspect}" } * ', '})"

data/lib/i18n/tasks/data/adapter/yaml_adapter.rb

@@ -5,11 +5,13 @@ module I18n::Tasks
   module Data
     module Adapter
       module YamlAdapter
+        EMOJI_REGEX = /\\u[\da-f]{8}/i
+
         class << self
           # @return [Hash] locale tree
           def parse(str, options)
             if YAML.method(:load).arity.abs == 2
-              YAML.load(str, **(options || {}))
+              YAML.safe_load(str, **(options || {}), permitted_classes: [Symbol], aliases: true)
             else
               # older jruby and rbx 2.2.7 do not accept options
               YAML.load(str)
@@ -18,7 +20,12 @@ module I18n::Tasks
 
           # @return [String]
           def dump(tree, options)
-            tree.to_yaml(options || {})
+            restore_emojis(tree.to_yaml(options || {}))
+          end
+
+          # @return [String]
+          def restore_emojis(yaml)
+            yaml.gsub(EMOJI_REGEX) { |m| [m[-8..].to_i(16)].pack("U") }
           end
         end
       end

data/lib/i18n/tasks/scanners/erb_ast_processor.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'ast'
+require 'set'
+require 'i18n/tasks/scanners/local_ruby_parser'
+
+module I18n::Tasks::Scanners
+  class ErbAstProcessor
+    include AST::Processor::Mixin
+    def initialize
+      super()
+      @ruby_parser = LocalRubyParser.new
+      @comments = []
+    end
+
+    def process_and_extract_comments(ast)
+      result = process(ast)
+      [result, @comments]
+    end
+
+    def on_code(node)
+      parsed, comments = @ruby_parser.parse(
+        node.children[0],
+        location: node.location
+      )
+      @comments.concat(comments)
+
+      unless parsed.nil?
+        parsed = parsed.updated(
+          nil,
+          parsed.children.map { |child| node?(child) ? process(child) : child }
+        )
+        node = node.updated(:send, parsed)
+      end
+      node
+    end
+
+    def handler_missing(node)
+      node.updated(
+        nil,
+        node.children.map { |child| node?(child) ? process(child) : child }
+      )
+    end
+
+    private
+
+    def node?(node)
+      node.is_a?(::Parser::AST::Node)
+    end
+  end
+end

data/lib/i18n/tasks/scanners/erb_ast_scanner.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks/scanners/ruby_ast_scanner'
+require 'i18n/tasks/scanners/erb_ast_processor'
+require 'better_html/errors'
+require 'better_html/parser'
+
+module I18n::Tasks::Scanners
+  # Scan for I18n.translate calls in ERB-file better-html and ASTs
+  class ErbAstScanner < RubyAstScanner
+    def initialize(**args)
+      super(**args)
+      @erb_ast_processor = ErbAstProcessor.new
+    end
+
+    private
+
+    # Parse file on path and returns AST and comments.
+    #
+    # @param path Path to file to parse
+    # @return [{Parser::AST::Node}, [Parser::Source::Comment]]
+    def path_to_ast_and_comments(path)
+      parser = BetterHtml::Parser.new(make_buffer(path))
+      ast = convert_better_html(parser.ast)
+      @erb_ast_processor.process_and_extract_comments(ast)
+    end
+
+    # Convert BetterHtml nodes to Parser::AST::Node
+    #
+    # @param node BetterHtml::Parser::AST::Node
+    # @return Parser::AST::Node
+    def convert_better_html(node)
+      definition = Parser::Source::Map::Definition.new(
+        node.location.begin,
+        node.location.begin,
+        node.location.begin,
+        node.location.end
+      )
+      Parser::AST::Node.new(
+        node.type,
+        node.children.map { |child| child.is_a?(BetterHtml::AST::Node) ? convert_better_html(child) : child },
+        {
+          location: definition
+        }
+      )
+    end
+  end
+end

data/lib/i18n/tasks/scanners/local_ruby_parser.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'parser/current'
+
+module I18n::Tasks::Scanners
+  class LocalRubyParser
+    def initialize
+      @parser = ::Parser::CurrentRuby.new
+    end
+
+    # Parse string and normalize location
+    def parse(source, location: nil)
+      buffer = ::Parser::Source::Buffer.new('(string)')
+      buffer.source = source
+
+      @parser.reset
+      ast, comments = @parser.parse_with_comments(buffer)
+      ast = normalize_location(ast, location)
+      comments = comments.map { |comment| normalize_comment_location(comment, location) }
+      [ast, comments]
+    end
+
+    # Normalize location for all parsed nodes
+
+    # @param node {Parser::AST::Node} Node in parsed code
+    # @param location {Parser::Source::Map} Global location for the parsed string
+    # @return {Parser::AST::Node}
+    def normalize_location(node, location)
+      return node.map { |child| normalize_location(child, location) } if node.is_a?(Array)
+
+      return node unless node.is_a?(::Parser::AST::Node)
+
+      node.updated(
+        nil,
+        node.children.map { |child| normalize_location(child, location) },
+        { location: updated_location(location, node.location) }
+      )
+    end
+
+    # Calculate location relative to a global location
+    #
+    # @param global_location {Parser::Source::Map} Global location where the code was parsed
+    # @param local_location {Parser::Source::Map} Local location in the parsed string
+    # @return {Parser::Source::Map}
+    def updated_location(global_location, local_location)
+      range = ::Parser::Source::Range.new(
+        global_location.expression.source_buffer,
+        global_location.expression.to_range.begin + local_location.expression.to_range.begin,
+        global_location.expression.to_range.begin + local_location.expression.to_range.end
+      )
+
+      ::Parser::Source::Map::Definition.new(
+        range.begin,
+        range.begin,
+        range.begin,
+        range.end
+      )
+    end
+
+    # Normalize location for comment
+    #
+    # @param comment {Parser::Source::Comment} A comment with local location
+    # @param location {Parser::Source::Map} Global location for the parsed string
+    # @return {Parser::Source::Comment}
+    def normalize_comment_location(comment, location)
+      range = ::Parser::Source::Range.new(
+        location.expression.source_buffer,
+        location.expression.to_range.begin + comment.location.expression.to_range.begin,
+        location.expression.to_range.begin + comment.location.expression.to_range.end
+      )
+      ::Parser::Source::Comment.new(range)
+    end
+  end
+end

data/lib/i18n/tasks/scanners/results/occurrence.rb

@@ -48,7 +48,7 @@ module I18n::Tasks
         # rubocop:enable Metrics/ParameterLists
 
         def inspect
-          "Occurrence(#{@path}:#{@line_num}:#{@line_pos}:#{@pos}:#{@raw_key}:#{@default_arg})"
+          "Occurrence(#{@path}:#{@line_num}:#{@line_pos}:#{@pos}:#{@raw_key}:#{@default_arg}:#{@line})"
         end
 
         def ==(other)

data/lib/i18n/tasks/scanners/ruby_ast_call_finder.rb

@@ -2,6 +2,7 @@
 
 require 'ast'
 require 'set'
+
 module I18n::Tasks::Scanners
   class RubyAstCallFinder
     include AST::Processor::Mixin

data/lib/i18n/tasks/scanners/ruby_ast_scanner.rb

@@ -32,32 +32,22 @@ module I18n::Tasks::Scanners
     #
     # @return [Array<[key, Results::KeyOccurrence]>] each occurrence found in the file
     def scan_file(path)
-      @parser.reset
-      ast, comments = @parser.parse_with_comments(make_buffer(path))
-
-      results = @call_finder.collect_calls ast do |send_node, method_name|
-        send_node_to_key_occurrence(send_node, method_name)
-      end
+      ast, comments = path_to_ast_and_comments(path)
 
-      magic_comments  = comments.select { |comment| comment.text =~ MAGIC_COMMENT_PREFIX }
-      comment_to_node = Parser::Source::Comment.associate_locations(ast, magic_comments).tap do |h|
-        # transform_values is only available in ActiveSupport 4.2+
-        h.each { |k, v| h[k] = v.first }
-      end.invert
-      results + (magic_comments.flat_map do |comment|
-        @parser.reset
-        associated_node = comment_to_node[comment]
-        @call_finder.collect_calls(
-          @parser.parse(make_buffer(path, comment.text.sub(MAGIC_COMMENT_PREFIX, '').split(/\s+(?=t)/).join('; ')))
-        ) do |send_node, _method_name|
-          # method_name is not available at this stage
-          send_node_to_key_occurrence(send_node, nil, location: associated_node || comment.location)
-        end
-      end)
+      ast_to_occurences(ast) + comments_to_occurences(path, ast, comments)
     rescue Exception => e # rubocop:disable Lint/RescueException
       raise ::I18n::Tasks::CommandError.new(e, "Error scanning #{path}: #{e.message}")
     end
 
+    # Parse file on path and returns AST and comments.
+    #
+    # @param path Path to file to parse
+    # @return [{Parser::AST::Node}, [Parser::Source::Comment]]
+    def path_to_ast_and_comments(path)
+      @parser.reset
+      @parser.parse_with_comments(make_buffer(path))
+    end
+
     # @param send_node [Parser::AST::Node]
     # @param method_name [Symbol, nil]
     # @param location [Parser::Source::Map]
@@ -204,6 +194,41 @@ module I18n::Tasks::Scanners
         buffer.raw_source = contents
       end
     end
+
+    # Convert an array of {Parser::Source::Comment} to occurrences.
+    #
+    # @param path Path to file
+    # @param ast Parser::AST::Node
+    # @param comments [Parser::Source::Comment]
+    # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
+    def comments_to_occurences(path, ast, comments)
+      magic_comments = comments.select { |comment| comment.text =~ MAGIC_COMMENT_PREFIX }
+      comment_to_node = Parser::Source::Comment.associate_locations(ast, magic_comments).tap do |h|
+        # transform_values is only available in ActiveSupport 4.2+
+        h.each { |k, v| h[k] = v.first }
+      end.invert
+
+      magic_comments.flat_map do |comment|
+        @parser.reset
+        associated_node = comment_to_node[comment]
+        @call_finder.collect_calls(
+          @parser.parse(make_buffer(path, comment.text.sub(MAGIC_COMMENT_PREFIX, '').split(/\s+(?=t)/).join('; ')))
+        ) do |send_node, _method_name|
+          # method_name is not available at this stage
+          send_node_to_key_occurrence(send_node, nil, location: associated_node || comment.location)
+        end
+      end
+    end
+
+    # Convert {Parser::AST::Node} to occurrences.
+    #
+    # @param ast {Parser::Source::Comment}
+    # @return [nil, [key, Occurrence]] full absolute key name and the occurrence.
+    def ast_to_occurences(ast)
+      @call_finder.collect_calls(ast) do |send_node, method_name|
+        send_node_to_key_occurrence(send_node, method_name)
+      end
+    end
   end
 end
 # rubocop:enable Metrics/AbcSize,Metrics/BlockNesting,Metrics/PerceivedComplexity

data/lib/i18n/tasks/scanners/ruby_key_literals.rb

@@ -2,12 +2,10 @@
 
 module I18n::Tasks::Scanners
   module RubyKeyLiterals
-    # NOTE
-    # "#{double_quoted["hash_pattern"]}" | "#{double_quoted_pattern}" | 'single_quoted_pattern' | :symbol_pattern
-    LITERAL_RE = /:?"[^\[]+\["[^"]+"\].+"|:?".+?"|:?'.+?'|:\w+/.freeze
+    LITERAL_RE = /:?".+?"|:?'.+?'|:\w+/.freeze
 
     # Match literals:
-    # * String: '', "#{}", "#{hash["key"]}"
+    # * String: '', "#{}"
     # * Symbol: :sym, :'', :"#{}"
     def literal_re
       LITERAL_RE
@@ -22,7 +20,7 @@ module I18n::Tasks::Scanners
       literal
     end
 
-    VALID_KEY_CHARS = /(?:[[:word:]]|[-.?!:;À-ž\/'"\[\]])/.freeze
+    VALID_KEY_CHARS = /(?:[[:word:]]|[-.?!:;À-ž\/])/.freeze
     VALID_KEY_RE    = /^#{VALID_KEY_CHARS}+$/.freeze
 
     def valid_key?(key)

data/lib/i18n/tasks/used_keys.rb

@@ -3,6 +3,7 @@
 require 'find'
 require 'i18n/tasks/scanners/pattern_with_scope_scanner'
 require 'i18n/tasks/scanners/ruby_ast_scanner'
+require 'i18n/tasks/scanners/erb_ast_scanner'
 require 'i18n/tasks/scanners/scanner_multiplexer'
 require 'i18n/tasks/scanners/files/caching_file_finder_provider'
 require 'i18n/tasks/scanners/files/caching_file_reader'
@@ -20,7 +21,8 @@ module I18n::Tasks
       relative_roots: %w[app/controllers app/helpers app/mailers app/presenters app/views].freeze,
       scanners: [
         ['::I18n::Tasks::Scanners::RubyAstScanner', { only: %w[*.rb] }],
-        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.rb] }]
+        ['::I18n::Tasks::Scanners::ErbAstScanner', { only: %w[*.erb] }],
+        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.erb *.rb] }]
       ],
       strict: true
     }.freeze

data/lib/i18n/tasks/version.rb

@@ -2,6 +2,6 @@
 
 module I18n
   module Tasks
-    VERSION = '0.9.36'
+    VERSION = '1.0.1'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: i18n-tasks
 version: !ruby/object:Gem::Version
-  version: 0.9.36
+  version: 1.0.1
 platform: ruby
 authors:
 - glebm
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-21 00:00:00.000000000 Z
+date: 2022-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: better_html
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: erubi
   requirement: !ruby/object:Gem::Requirement
@@ -357,12 +371,15 @@ files:
 - lib/i18n/tasks/references.rb
 - lib/i18n/tasks/reports/base.rb
 - lib/i18n/tasks/reports/terminal.rb
+- lib/i18n/tasks/scanners/erb_ast_processor.rb
+- lib/i18n/tasks/scanners/erb_ast_scanner.rb
 - lib/i18n/tasks/scanners/file_scanner.rb
 - lib/i18n/tasks/scanners/files/caching_file_finder.rb
 - lib/i18n/tasks/scanners/files/caching_file_finder_provider.rb
 - lib/i18n/tasks/scanners/files/caching_file_reader.rb
 - lib/i18n/tasks/scanners/files/file_finder.rb
 - lib/i18n/tasks/scanners/files/file_reader.rb
+- lib/i18n/tasks/scanners/local_ruby_parser.rb
 - lib/i18n/tasks/scanners/occurrence_from_position.rb
 - lib/i18n/tasks/scanners/pattern_mapper.rb
 - lib/i18n/tasks/scanners/pattern_scanner.rb
@@ -406,7 +423,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.6'
   - - "<"
     - !ruby/object:Gem::Version
       version: '4.0'