i18n-youdao-tasks 0.9.37

data/lib/i18n/tasks/key_pattern_matching.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'strscan'
+
+module I18n::Tasks::KeyPatternMatching
+  extend self # rubocop:disable Style/ModuleFunction
+
+  MATCH_NOTHING = /\z\A/.freeze
+
+  # one regex to match any
+  def compile_patterns_re(key_patterns)
+    if key_patterns.blank?
+      # match nothing
+      MATCH_NOTHING
+    else
+      /(?:#{key_patterns.map { |p| compile_key_pattern p } * '|'})/m
+    end
+  end
+
+  # convert pattern to regex
+  # In patterns:
+  #      *     is like .* in regexs
+  #      :     matches a single key
+  #   { a, b.c } match any in set, can use : and *, match is captured
+  def compile_key_pattern(key_pattern)
+    return key_pattern if key_pattern.is_a?(Regexp)
+
+    /\A#{key_pattern_re_body(key_pattern)}\z/
+  end
+
+  def key_pattern_re_body(key_pattern)
+    key_pattern
+      .gsub(/\./, '\.')
+      .gsub(/\*/, '.*')
+      .gsub(/:/, '(?<=^|\.)[^.]+?(?=\.|$)')
+      .gsub(/\{(.*?)}/) { "(#{Regexp.last_match(1).strip.gsub(/\s*,\s*/, '|')})" }
+  end
+end

data/lib/i18n/tasks/locale_list.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module I18n::Tasks
+  module LocaleList
+    module_function
+
+    # @return locales converted to strings, with base locale first, the rest sorted alphabetically
+    def normalize_locale_list(locales, base_locale, include_base = false)
+      locales = Array(locales).map(&:to_s).sort
+      if locales.include?(base_locale)
+        [base_locale] + (locales - [base_locale])
+      elsif include_base
+        [base_locale] + locales
+      else
+        locales
+      end
+    end
+  end
+end

data/lib/i18n/tasks/locale_pathname.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module I18n::Tasks
+  module LocalePathname
+    class << self
+      def replace_locale(path, from, to)
+        path&.gsub(path_locale_re(from), to)
+      end
+
+      private
+
+      def path_locale_re(locale)
+        (@path_locale_res ||= {})[locale] ||= %r{(?<=^|[/.])#{locale}(?=[/.])}
+      end
+    end
+  end
+end

data/lib/i18n/tasks/logging.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module I18n::Tasks::Logging
+  module_function
+
+  MUTEX = Mutex.new
+  PROGRAM_NAME = File.basename($PROGRAM_NAME)
+
+  def warn_deprecated(message)
+    log_stderr Rainbow("#{program_name}: [DEPRECATED] #{message}").yellow.bright
+  end
+
+  def log_verbose(message = nil)
+    log_stderr Rainbow(message || yield).blue.bright if ::I18n::Tasks.verbose?
+  end
+
+  def log_warn(message)
+    log_stderr Rainbow("#{program_name}: [WARN] #{message}").yellow
+  end
+
+  def log_error(message)
+    log_stderr Rainbow("#{program_name}: #{message}").red.bright
+  end
+
+  def log_stderr(*args)
+    # We don't want output from different threads to get intermixed.
+    MUTEX.synchronize do
+      $stderr.puts(*args)
+    end
+  end
+
+  def program_name
+    PROGRAM_NAME
+  end
+end

data/lib/i18n/tasks/missing_keys.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require 'set'
+module I18n::Tasks
+  module MissingKeys # rubocop:disable Metrics/ModuleLength
+    MISSING_TYPES = %w[
+      used
+      diff
+      plural
+    ].freeze
+
+    def self.missing_keys_types
+      @missing_keys_types ||= MISSING_TYPES
+    end
+
+    def missing_keys_types
+      MissingKeys.missing_keys_types
+    end
+
+    # @param types [:used, :diff, :plural] all if `nil`.
+    # @return [Siblings]
+    def missing_keys(locales: nil, types: nil, base_locale: nil)
+      locales ||= self.locales
+      types   ||= missing_keys_types
+      base = base_locale || self.base_locale
+      types.inject(empty_forest) do |f, type|
+        f.merge! send(:"missing_#{type}_forest", locales, base)
+      end
+    end
+
+    def eq_base_keys(opts = {})
+      locales = Array(opts[:locales]).presence || self.locales
+      (locales - [base_locale]).inject(empty_forest) do |tree, locale|
+        tree.merge! equal_values_tree(locale, base_locale)
+      end
+    end
+
+    def missing_diff_forest(locales, base = base_locale)
+      tree = empty_forest
+      # present in base but not locale
+      (locales - [base]).each do |locale|
+        tree.merge! missing_diff_tree(locale, base)
+      end
+      if locales.include?(base)
+        # present in locale but not base
+        (self.locales - [base]).each do |locale|
+          tree.merge! missing_diff_tree(base, locale)
+        end
+      end
+      tree
+    end
+
+    def missing_used_forest(locales, _base = base_locale)
+      locales.inject(empty_forest) do |forest, locale|
+        forest.merge! missing_used_tree(locale)
+      end
+    end
+
+    def missing_plural_forest(locales, _base = base_locale)
+      locales.each_with_object(empty_forest) do |locale, forest|
+        required_keys = required_plural_keys_for_locale(locale)
+        next if required_keys.empty?
+
+        tree = empty_forest
+        plural_nodes data[locale] do |node|
+          children = node.children
+          present_keys = Set.new(children.map { |c| c.key.to_sym })
+          next if ignore_key?(node.full_key(root: false), :missing)
+          next if present_keys.superset?(required_keys)
+
+          tree[node.full_key] = node.derive(
+            value: children.to_hash,
+            children: nil,
+            data: node.data.merge(missing_keys: (required_keys - present_keys).to_a)
+          )
+        end
+        tree.set_root_key!(locale, type: :missing_plural)
+        forest.merge!(tree)
+      end
+    end
+
+    def required_plural_keys_for_locale(locale)
+      @plural_keys_for_locale ||= {}
+      return @plural_keys_for_locale[locale] if @plural_keys_for_locale.key?(locale)
+
+      @plural_keys_for_locale[locale] = plural_keys_for_locale(locale)
+    end
+
+    # Loads rails-i18n pluralization config for the given locale.
+    def load_rails_i18n_pluralization!(locale)
+      path = File.join(Gem::Specification.find_by_name('rails-i18n').gem_dir, 'rails', 'pluralization', "#{locale}.rb")
+      eval(File.read(path), binding, path) # rubocop:disable Security/Eval
+    end
+
+    # keys present in compared_to, but not in locale
+    def missing_diff_tree(locale, compared_to = base_locale)
+      data[compared_to].select_keys do |key, _node|
+        locale_key_missing? locale, depluralize_key(key, compared_to)
+      end.set_root_key!(locale, type: :missing_diff).keys do |_key, node|
+        # change path and locale to base
+        data = { locale: locale, missing_diff_locale: node.data[:locale] }
+        if node.data.key?(:path)
+          data[:path] = LocalePathname.replace_locale(node.data[:path], node.data[:locale], locale)
+        end
+        node.data.update data
+      end
+    end
+
+    # keys used in the code missing translations in locale
+    def missing_used_tree(locale)
+      used_tree(strict: true).select_keys do |key, _node|
+        locale_key_missing?(locale, key)
+      end.set_root_key!(locale, type: :missing_used)
+    end
+
+    def equal_values_tree(locale, compare_to = base_locale)
+      base = data[compare_to].first.children
+      data[locale].select_keys(root: false) do |key, node|
+        other_node = base[key]
+        other_node && !node.reference? && node.value == other_node.value && !ignore_key?(key, :eq_base, locale)
+      end.set_root_key!(locale, type: :eq_base)
+    end
+
+    def locale_key_missing?(locale, key)
+      !key_value?(key, locale) && !external_key?(key, locale) && !ignore_key?(key, :missing)
+    end
+
+    # @param [::I18n::Tasks::Data::Tree::Siblings] forest
+    # @yield [::I18n::Tasks::Data::Tree::Node]
+    # @yieldreturn [Boolean] whether to collapse the node
+    def collapse_same_key_in_locales!(forest)
+      locales_and_node_by_key = {}
+      to_remove               = []
+      forest.each do |root|
+        locale = root.key
+        root.keys do |key, node|
+          next unless yield node
+
+          if locales_and_node_by_key.key?(key)
+            locales_and_node_by_key[key][0] << locale
+          else
+            locales_and_node_by_key[key] = [[locale], node]
+          end
+          to_remove << node
+        end
+      end
+      forest.remove_nodes_and_emptied_ancestors! to_remove
+      locales_and_node_by_key.each_with_object({}) do |(key, (locales, node)), inv|
+        (inv[locales.sort.join('+')] ||= []) << [key, node]
+      end.map do |locales, keys_nodes|
+        keys_nodes.each do |(key, node)|
+          forest["#{locales}.#{key}"] = node
+        end
+      end
+      forest
+    end
+
+    private
+
+    def plural_keys_for_locale(locale)
+      configuration = load_rails_i18n_pluralization!(locale)
+      if configuration[locale.to_sym].nil?
+        alternate_locale = alternate_locale_from(locale)
+        return Set.new if configuration[alternate_locale.to_sym].nil?
+
+        return set_from_rails_i18n_pluralization(configuration, alternate_locale)
+      end
+      set_from_rails_i18n_pluralization(configuration, locale)
+    rescue SystemCallError, IOError
+      Set.new
+    end
+
+    def alternate_locale_from(locale)
+      re = /(\w{2})-*(\w{2,3})*/
+      match = locale.match(re)
+      language_code = match[1]
+      country_code = match[2]
+      "#{language_code}-#{country_code.upcase}"
+    end
+
+    def set_from_rails_i18n_pluralization(configuration, locale)
+      Set.new(configuration[locale.to_sym][:i18n][:plural][:keys])
+    end
+  end
+end

data/lib/i18n/tasks/plural_keys.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'set'
+module I18n::Tasks::PluralKeys
+  # Ref: http://cldr.unicode.org/index/cldr-spec/plural-rules
+  CLDR_CATEGORY_KEYS = %w[zero one two few many other].freeze
+  PLURAL_KEY_SUFFIXES = Set.new CLDR_CATEGORY_KEYS
+  PLURAL_KEY_RE = /\.(?:#{CLDR_CATEGORY_KEYS * '|'})$/.freeze
+
+  def collapse_plural_nodes!(tree)
+    tree.leaves.map(&:parent).compact.uniq.each do |node|
+      children = node.children
+      next unless plural_forms?(children)
+
+      node.value    = children.to_hash
+      node.children = nil
+      node.data.merge! children.first.data
+    end
+    tree
+  end
+
+  # @param [String] key i18n key
+  # @param [String] locale to pull key data from
+  # @return [String] the base form if the key is a specific plural form (e.g. apple for apple.many), the key otherwise.
+  def depluralize_key(key, locale = base_locale)
+    return key if key !~ PLURAL_KEY_RE
+
+    key_name = last_key_part(key)
+    parent_key = key[0..- (key_name.length + 2)]
+    nodes = tree("#{locale}.#{parent_key}").presence || (locale != base_locale && tree("#{base_locale}.#{parent_key}"))
+    if nodes && plural_forms?(nodes)
+      parent_key
+    else
+      key
+    end
+  end
+
+  # @param [::I18n::Tasks::Data::Tree::Traversal] tree
+  # @yieldparam node [::I18n::Tasks::Data::Tree::Node] plural node
+  def plural_nodes(tree)
+    return to_enum(:plural_nodes, tree) unless block_given?
+
+    visited = Set.new
+    tree.leaves do |node|
+      parent = node.parent
+      next if !parent || visited.include?(parent)
+
+      yield parent if plural_forms?(parent.children)
+      visited.add(parent)
+    end
+    self
+  end
+
+  def plural_forms?(s)
+    return false if non_plural_other?(s)
+
+    s.present? && s.all? { |node| node.leaf? && plural_suffix?(node.key) }
+  end
+
+  def non_plural_other?(s)
+    s.size == 1 && s.first.leaf? && (!s.first.value.is_a?(String) || !s.first.value.include?('%{count}'))
+  end
+
+  def plural_suffix?(key)
+    PLURAL_KEY_SUFFIXES.include?(key)
+  end
+end

data/lib/i18n/tasks/references.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module I18n::Tasks
+  module References
+    # Given a raw usage tree and a tree of reference keys in the data, return 3 trees:
+    # 1. Raw references -- a subset of the usages tree with keys that are reference key usages.
+    # 2. Resolved references -- all the used references in their fully resolved form.
+    # 3. Reference keys -- all the used reference keys.
+    def process_references(usages,
+                           data_refs = merge_reference_trees(data_forest.select_keys { |_, node| node.reference? }))
+      fail ArgumentError, 'usages must be a Data::Tree::Instance' unless usages.is_a?(Data::Tree::Siblings)
+      fail ArgumentError, 'all_references must be a Data::Tree::Instance' unless data_refs.is_a?(Data::Tree::Siblings)
+
+      raw_refs = empty_forest
+      resolved_refs = empty_forest
+      refs = empty_forest
+      data_refs.key_to_node.each do |ref_key_part, ref_node|
+        usages.each do |usage_node|
+          next unless usage_node.key == ref_key_part
+
+          if ref_node.leaf?
+            process_leaf!(ref_node, usage_node, raw_refs, resolved_refs, refs)
+          else
+            process_non_leaf!(ref_node, usage_node, raw_refs, resolved_refs, refs)
+          end
+        end
+      end
+      [raw_refs, resolved_refs, refs]
+    end
+
+    private
+
+    # @param [I18n::Tasks::Data::Tree::Node] ref
+    # @param [I18n::Tasks::Data::Tree::Node] usage
+    # @param [I18n::Tasks::Data::Tree::Siblings] raw_refs
+    # @param [I18n::Tasks::Data::Tree::Siblings] resolved_refs
+    # @param [I18n::Tasks::Data::Tree::Siblings] refs
+    def process_leaf!(ref, usage, raw_refs, resolved_refs, refs)
+      refs.merge_node!(Data::Tree::Node.new(key: ref.key, data: usage.data)) unless refs.key_to_node.key?(ref.key)
+      new_resolved_refs = Data::Tree::Siblings.from_key_names([ref.value.to_s]) do |_, resolved_node|
+        raw_refs.merge_node!(usage)
+        if usage.leaf?
+          resolved_node.data.merge!(usage.data)
+        else
+          resolved_node.children = usage.children
+        end
+        resolved_node.leaves { |node| node.data[:ref_info] = [ref.full_key, ref.value.to_s] }
+      end
+      add_occurences! refs.key_to_node[ref.key].data, new_resolved_refs
+      resolved_refs.merge! new_resolved_refs
+    end
+
+    # @param [Hash] ref_data
+    # @param [I18n::Tasks::Data::Tree::Siblings] new_resolved_refs
+    def add_occurences!(ref_data, new_resolved_refs)
+      ref_data[:occurrences] ||= []
+      new_resolved_refs.leaves do |leaf|
+        ref_data[:occurrences].concat(leaf.data[:occurrences] || [])
+      end
+      ref_data[:occurrences].sort_by!(&:path)
+      ref_data[:occurrences].uniq!
+    end
+
+    # @param [I18n::Tasks::Data::Tree::Node] ref
+    # @param [I18n::Tasks::Data::Tree::Node] usage
+    # @param [I18n::Tasks::Data::Tree::Siblings] raw_refs
+    # @param [I18n::Tasks::Data::Tree::Siblings] resolved_refs
+    # @param [I18n::Tasks::Data::Tree::Siblings] refs
+    def process_non_leaf!(ref, usage, raw_refs, resolved_refs, refs)
+      child_raw_refs, child_resolved_refs, child_refs = process_references(usage.children, ref.children)
+      raw_refs.merge_node! Data::Tree::Node.new(key: ref.key, children: child_raw_refs) unless child_raw_refs.empty?
+      resolved_refs.merge! child_resolved_refs
+      refs.merge_node! Data::Tree::Node.new(key: ref.key, children: child_refs) unless child_refs.empty?
+    end
+
+    # Given a forest of references, merge trees into one tree, ensuring there are no conflicting references.
+    # @param roots [I18n::Tasks::Data::Tree::Siblings]
+    # @return [I18n::Tasks::Data::Tree::Siblings]
+    def merge_reference_trees(roots)
+      roots.inject(empty_forest) do |forest, root|
+        root.keys do |full_key, node|
+          if full_key == node.value.to_s
+            log_warn(
+              "Self-referencing key #{node.full_key(root: false).inspect} in #{node.data[:locale].inspect}"
+            )
+          end
+        end
+        forest.merge!(
+          root.children,
+          on_leaves_merge: lambda do |node, other|
+            if node.value != other.value
+              log_warn(
+                'Conflicting references: '\
+            "#{node.full_key(root: false)} ⮕ #{node.value} in #{node.data[:locale]},"\
+            " but ⮕ #{other.value} in #{other.data[:locale]}"
+              )
+            end
+          end
+        )
+      end
+    end
+  end
+end

data/lib/i18n/tasks/reports/base.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module I18n::Tasks::Reports
+  class Base
+    include I18n::Tasks::Logging
+
+    def initialize(task = I18n::Tasks::BaseTask.new)
+      @task = task
+    end
+
+    attr_reader :task
+
+    delegate :base_locale, :locales, to: :task
+
+    protected
+
+    def missing_type_info(type)
+      ::I18n::Tasks::MissingKeys::MISSING_TYPES[type.to_s.sub(/\Amissing_/, '').to_sym]
+    end
+
+    def missing_title(forest)
+      "Missing translations (#{forest.leaves.count || '∅'})"
+    end
+
+    def inconsistent_interpolations_title(forest)
+      "Inconsistent interpolations (#{forest.leaves.count || '∅'})"
+    end
+
+    def unused_title(key_values)
+      "Unused keys (#{key_values.count || '∅'})"
+    end
+
+    def eq_base_title(key_values, locale = base_locale)
+      "Same value as #{locale} (#{key_values.count || '∅'})"
+    end
+
+    def used_title(keys_nodes, filter)
+      used_n = keys_nodes.map { |_k, node| node.data[:occurrences].size }.reduce(:+).to_i
+      "#{keys_nodes.size} key#{'s' if keys_nodes.size != 1}#{" matching '#{filter}'" if filter}"\
+      "#{" (#{used_n} usage#{'s' if used_n != 1})" if used_n.positive?}"
+    end
+
+    # Sort keys by their attributes in order
+    # @param [Hash] order e.g. {locale: :asc, type: :desc, key: :asc}
+    def sort_by_attr!(objects, order = { locale: :asc, key: :asc })
+      order_keys = order.keys
+      objects.sort! do |a, b|
+        by = order_keys.detect { |k| a[k] != b[k] }
+        order[by] == :desc ? b[by] <=> a[by] : a[by] <=> b[by]
+      end
+      objects
+    end
+
+    def forest_to_attr(forest)
+      forest.keys(root: false).map do |key, node|
+        { key: key, value: node.value, type: node.data[:type], locale: node.root.key, data: node.data }
+      end
+    end
+
+    def format_locale(locale)
+      return '' unless locale
+
+      if locale.split('+') == task.locales.sort
+        'all'
+      else
+        locale.tr '+', ' '
+      end
+    end
+
+    def collapse_missing_tree!(forest)
+      forest = task.collapse_plural_nodes!(forest)
+      task.collapse_same_key_in_locales!(forest) { |node| node.data[:type] == :missing_used }
+    end
+  end
+end