i18n-processes 0.1.0

data/lib/i18n/processes/reports/spreadsheet.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/reports/base'
+require 'fileutils'
+require 'i18n/processes/path'
+
+module I18n::Processes::Reports
+  class Spreadsheet < Base
+    include I18n::Processes::Path
+
+    def missing_report(locale)
+      path = 'tmp/missing_keys/'
+      FileUtils::mkdir_p path  unless Dir.exist?path
+      file = "#{path}missing_keys_#{locale}"
+      report = File.new(file, 'w')
+      report.write("# 说明：以#开头的行，表示key对应的中文翻译\n# 下一行'='左边为key，'='右边需要填上对应的#{locale}翻译： \n")
+      report.write("\n\n# =======================  missing keys list =============================\n\n")
+      find_missing(locale).map do |k,v|
+        report.write("# #{v}")
+        report.write("#{k}=#{k}\n\n")
+      end
+      report.close
+      $stderr.puts Rainbow("missing report saved to #{file}\n").green
+    end
+
+    def translated_files(locale)
+      path = translated_path.first
+      dic = get_dic("./tmp/#{locale}")
+      FileUtils.rm_f Dir.glob("./#{path}**/**") unless Dir["./#{path}**/**"].size.zero?
+      origin_files = origin_files(base_locale).flatten
+      # $stderr.puts Rainbow origin_files
+      origin_files.each do |origin_file|
+        translated_file(origin_file,"#{path}#{locale}/",  dic)
+      end
+      $stderr.puts Rainbow("translated files saved to #{path}\n").green
+    end
+
+    def find_missing(locale = nil)
+      path = './tmp/'
+      comp_dic = get_dic(path + locale)
+      base_dic = get_dic(path + base_locale)
+      base_dic.select { |k,v| (base_dic.keys - comp_dic.keys).include?(k)}
+    end
+
+    private
+
+    def translated_file(file, path, dic)
+      translated_file = new_file(file, path)
+      File.open(file).read.each_line do |line|
+        if line =~ /^#/ || line == "\n" || !line.include?('.')
+          translated_file.write line
+        elsif line.include?('\':')
+          line.gsub!(/'|,/, '')
+          key = line.split(': ').first.delete(' ')
+          translated_file.write("  '#{key}': '#{dic[key].chomp}',\n") if dic.key?(key)
+        else
+          key = line.split('=').first.delete(' ')
+          translated_file.write("#{key}=#{dic[key]}") if dic.key?(key)
+        end
+      end
+      translated_file.close
+    end
+
+    def new_file(file, path)
+      sourced = source_path.first
+      new_file = file.sub(sourced, path) if file.include?(sourced)
+      FileUtils::mkdir_p File.dirname(new_file) unless Dir.exist?File.dirname(new_file)
+      File.new(new_file, 'w')
+    end
+
+  end
+end

data/lib/i18n/processes/reports/terminal.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/reports/base'
+require 'i18n/processes/rainbow_utils'
+require 'terminal-table'
+module I18n
+  module Processes
+    module Reports
+      class Terminal < Base # rubocop:disable Metrics/ClassLength
+        def missing_keys(forest = task.missing_keys)
+          forest = collapse_missing_tree! forest
+          if forest.present?
+            print_title missing_title(forest)
+            print_table headings: [Rainbow('Locale').cyan.bright,
+                                   Rainbow('Key').cyan.bright,
+                                   'Value in other locales or source'] do |t|
+              t.rows = sort_by_attr!(forest_to_attr(forest)).map do |a|
+                [{ value: Rainbow(format_locale(a[:locale])).cyan, alignment: :center },
+                 format_key(a[:key], a[:data]),
+                 missing_key_info(a)]
+              end
+            end
+          else
+            print_success 'No translations are missing.'
+          end
+        end
+
+        def changed_keys(diff = nil)
+          if diff
+            print_title "#{diff.count} keys' value changed"
+            print_table headings: [Rainbow('key').cyan.bright,
+                                   Rainbow('Current').cyan.bright,
+                                   'Previous'] do |t|
+              t.rows = diff.map do |key, value|
+                [key, value[:current], value[:previous]]
+              end
+            end
+          else
+            print_success 'No key have been changed.'
+          end
+        end
+
+        def icon(type)
+          glyph = missing_type_info(type)[:glyph]
+          { missing_used: Rainbow(glyph).red, missing_diff: Rainbow(glyph).yellow }[type]
+        end
+
+        def used_keys(used_tree = task.used_tree)
+          # For the used tree we may have usage nodes that are not leaves as references.
+          keys_nodes = used_tree.nodes.select { |node| node.data[:occurrences].present? }.map do |node|
+            [node.full_key(root: false), node]
+          end
+          print_title used_title(keys_nodes, used_tree.first.root.data[:key_filter])
+          # Group multiple nodes
+          if keys_nodes.present?
+            keys_nodes.sort! { |a, b| a[0] <=> b[0] }.each do |key, node|
+              print_occurrences node, key
+            end
+          else
+            print_error 'No key usages found.'
+          end
+        end
+
+        def unused_keys(tree = task.unused_keys)
+          keys = tree.root_key_value_data(true)
+          if keys.present?
+            print_title unused_title(keys)
+            print_locale_key_value_data_table keys
+          else
+            print_success 'Every translation is in use.'
+          end
+        end
+
+        def eq_base_keys(tree = task.eq_base_keys)
+          keys = tree.root_key_value_data(true)
+          if keys.present?
+            print_title eq_base_title(keys)
+            print_locale_key_value_data_table keys
+          else
+            print_info Rainbow('No translations are the same as base value').cyan
+          end
+        end
+
+        def show_tree(tree)
+          print_locale_key_value_data_table tree.root_key_value_data(true)
+        end
+
+        def forest_stats(forest, stats = task.forest_stats(forest))
+          text  = if stats[:locale_count] == 1
+                    "has #{stats[:key_count]} keys in total. On average, values are #{stats[:value_chars_avg]} characters long, keys have #{stats[:key_segments_avg]} segments."
+                  else
+                    "has #{stats[:key_count]} keys across #{stats[:locale_count]} locales. On average, values are #{stats[:value_chars_avg]} characters long, keys have #{stats[:key_segments_avg]} segments, a locale has #{stats[:per_locale_avg]} keys."
+                  end
+          title = Rainbow("Forest (#{stats.slice(:locales)})").bright
+          print_info "#{Rainbow(title).cyan} #{Rainbow(text).cyan}"
+        end
+
+        def mv_results(results)
+          results.each do |(from, to)|
+            if to
+              print_info "#{Rainbow(from).cyan} #{Rainbow('⮕').yellow.bright} #{Rainbow(to).cyan}"
+            else
+              print_info "#{Rainbow(from).red}#{Rainbow(' 🗑').red.bright}"
+            end
+          end
+        end
+
+        def check_normalized_results(non_normalized)
+          if non_normalized.empty?
+            print_success 'All data is normalized'
+            return
+          end
+          log_stderr Rainbow('The following data requires normalization:').yellow
+          puts non_normalized
+          log_stderr Rainbow('Run `i18n-processes normalize` to fix').yellow
+        end
+
+        private
+
+        def missing_key_info(leaf)
+          if leaf[:type] == :missing_used
+            first_occurrence leaf
+          else
+            "#{Rainbow(leaf[:data][:missing_diff_locale]).cyan} "\
+            "#{format_value(leaf[:value].is_a?(String) ? leaf[:value].strip : leaf[:value])}"
+          end
+        end
+
+        def format_key(key, data)
+          if data[:ref_info]
+            from, to = data[:ref_info]
+            resolved = key[0...to.length]
+            after    = key[to.length..-1]
+            "  #{Rainbow(from).yellow}#{Rainbow(after).cyan}\n" \
+            "#{Rainbow('⮕').yellow.bright} #{Rainbow(resolved).yellow.bright}"
+          else
+            Rainbow(key).cyan
+          end
+        end
+
+        def format_value(val)
+          val.is_a?(Symbol) ? "#{Rainbow('⮕ ').yellow.bright}#{Rainbow(val).yellow}" : val.to_s.strip
+        end
+
+        def format_reference_desc(node_data)
+          return nil unless node_data
+          case node_data[:ref_type]
+          when :reference_usage
+            Rainbow('(ref)').yellow.bright
+          when :reference_usage_resolved
+            Rainbow('(resolved ref)').yellow.bright
+          when :reference_usage_key
+            Rainbow('(ref key)').yellow.bright
+          end
+        end
+
+        def print_occurrences(node, full_key = node.full_key)
+          occurrences = node.data[:occurrences]
+          puts [Rainbow(full_key).bright,
+                format_reference_desc(node.data),
+                (Rainbow(occurrences.size).green if occurrences.size > 1)].compact.join ' '
+          occurrences.each do |occurrence|
+            puts "  #{key_occurrence full_key, occurrence}"
+          end
+        end
+
+        def print_locale_key_value_data_table(locale_key_value_datas)
+          if locale_key_value_datas.present?
+            print_table headings: [Rainbow("Locale").cyan.bright,
+                                   Rainbow("Key").cyan.bright,
+                                   "Value"] do |t|
+              t.rows = locale_key_value_datas.map { |(locale, k, v, data)|
+                [{ value: Rainbow(locale).cyan, alignment: :center }, format_key(k, data), format_value(v)]
+              }
+            end
+          else
+            puts 'ø'
+          end
+        end
+
+        def print_title(title)
+          log_stderr "#{Rainbow(title.strip).bright} #{I18n::Processes::RainbowUtils.faint_color('|')} " \
+                     "#{"i18n-tasks v#{I18n::Processes::VERSION}"}"
+        end
+
+        def print_success(message)
+          log_stderr Rainbow("✓ #{["Good job!", "Well done!", "Perfect!"].sample} #{message}").green.bright
+        end
+
+        def print_error(message)
+          log_stderr(Rainbow(message).red.bright)
+        end
+
+        def print_info(message)
+          log_stderr message
+        end
+
+        def indent(txt, n = 2)
+          txt.gsub(/^/, ' ' * n)
+        end
+
+        def print_table(opts, &block)
+          puts ::Terminal::Table.new(opts, &block)
+        end
+
+        def key_occurrence(full_key, occurrence)
+          location = Rainbow("#{occurrence.path}:#{occurrence.line_num}").green
+          source   = highlight_key(occurrence.raw_key || full_key, occurrence.line, occurrence.line_pos..-1).strip
+          "#{location} #{source}"
+        end
+
+        def first_occurrence(leaf)
+          # @type [I18n::Processes::Scanners::KeyOccurrences]
+          occurrences = leaf[:data][:occurrences]
+          # @type [I18n::Processes::Scanners::Occurrence]
+          first = occurrences.first
+          [
+            Rainbow("#{first.path}:#{first.line_num}").green,
+            ("(#{occurrences.length - 1} more)" if occurrences.length > 1)
+
+          ].compact.join(' ')
+        end
+
+        def highlight_key(full_key, line, range = (0..-1))
+          line.dup.tap do |s|
+            s[range] = s[range].sub(full_key) do |m|
+              highlight_string m
+            end
+          end
+        end
+
+        module HighlightUnderline
+          def highlight_string(s)
+            Rainbow(s).underline
+          end
+        end
+
+        module HighlightOther
+          def highlight_string(s)
+            Rainbow(s).yellow
+          end
+        end
+
+        if Gem.win_platform?
+          include HighlightOther
+        else
+          include HighlightUnderline
+        end
+      end
+    end
+  end
+end

data/lib/i18n/processes/scanners/file_scanner.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/scanners/scanner'
+
+module I18n::Processes::Scanners
+  # A base class for a scanner that analyses files.
+  #
+  # @abstract The child must implement {#scan_file}.
+  # @since 0.9.0
+  class FileScanner < Scanner
+    attr_reader :config
+
+    def initialize(
+        config: {},
+        file_finder_provider: Files::CachingFileFinderProvider.new,
+        file_reader: Files::CachingFileReader.new
+    )
+      @config      = config
+      @file_reader = file_reader
+      @file_finder = file_finder_provider.get(**config.slice(:paths, :only, :exclude))
+    end
+
+    # @return (see Scanner#keys)
+    def keys
+      (traverse_files do |path|
+        scan_file(path)
+      end.reduce(:+) || []).group_by(&:first).map do |key, keys_occurrences|
+        Results::KeyOccurrences.new(key: key, occurrences: keys_occurrences.map(&:second))
+      end
+    end
+
+    protected
+
+    # Extract all occurrences of translate calls from the file at the given path.
+    #
+    # @return [Array<[key, Results::KeyOccurrence]>] each occurrence found in the file
+    def scan_file(_path)
+      fail 'Unimplemented'
+    end
+
+    # Read a file. Reads of the same path are cached.
+    #
+    # @param path [String]
+    # @return [String] file contents
+    def read_file(path)
+      @file_reader.read_file(path)
+    end
+
+    # Traverse the paths and yield the matching ones.
+    #
+    # @note This method is cached, it will only access the filesystem on the first invocation.
+    # @param (see FileFinder#traverse_files)
+    # @yieldparam (see FileFinder#traverse_files)
+    # @return (see FileFinder#traverse_files)
+    def traverse_files(&block)
+      @file_finder.traverse_files(&block)
+    end
+
+    # @note This method is cached, it will only access the filesystem on the first invocation.
+    # @return (see FileFinder#find_files)
+    def find_files
+      @file_finder.find_files
+    end
+  end
+end

data/lib/i18n/processes/scanners/files/caching_file_finder.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/scanners/files/file_finder'
+module I18n::Processes::Scanners::Files
+  # Finds the files in the specified search paths with support for exclusion / inclusion patterns.
+  # Wraps a {FileFinder} and caches the results.
+  #
+  # @note This class is thread-safe. All methods are cached.
+  # @since 0.9.0
+  class CachingFileFinder < FileFinder
+    # @param (see FileFinder#initialize)
+    def initialize(**args)
+      super
+      @mutex = Mutex.new
+      @cached_paths = nil
+    end
+
+    # Traverse the paths and yield the matching ones.
+    #
+    # @note This method is cached, it will only access the filesystem on the first invocation.
+    # @param (see FileFinder#traverse_files)
+    # @yieldparam (see FileFinder#traverse_files)
+    # @return (see FileFinder#traverse_files)
+    def traverse_files
+      super
+    end
+
+    # @note This method is cached, it will only access the filesystem on the first invocation.
+    # @return (see FileFinder#find_files)
+    def find_files
+      @cached_paths || @mutex.synchronize { @cached_paths ||= super }
+    end
+  end
+end

data/lib/i18n/processes/scanners/files/caching_file_finder_provider.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/scanners/files/caching_file_finder'
+
+module I18n::Processes::Scanners::Files
+  # Finds the files and provides their contents.
+  #
+  # @note This class is thread-safe. All methods are cached.
+  # @since 0.9.0
+  class CachingFileFinderProvider
+    # @param exclude [Array<String>]
+    def initialize(exclude: [])
+      @cache = {}
+      @mutex = Mutex.new
+      @defaults = { exclude: exclude }
+    end
+
+    # Initialize a {CachingFileFinder} or get one from cache based on the constructor arguments.
+    #
+    # @param (see FileFinder#initialize)
+    # @return [CachingFileFinder]
+    def get(**file_finder_args)
+      @cache[file_finder_args] || @mutex.synchronize do
+        @cache[file_finder_args] ||= begin
+          args = file_finder_args.dup
+          args[:exclude] = @defaults[:exclude] + (args[:exclude] || [])
+          args[:exclude].uniq!
+          CachingFileFinder.new(**args)
+        end
+      end
+    end
+  end
+end

data/lib/i18n/processes/scanners/files/caching_file_reader.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/scanners/files/file_reader'
+module I18n::Processes::Scanners::Files
+  # Reads the files in 'rb' mode and UTF-8 encoding.
+  # Wraps a {FileReader} and caches the results.
+  #
+  # @note This class is thread-safe. All methods are cached.
+  # @since 0.9.0
+  class CachingFileReader < FileReader
+    def initialize
+      super
+      @mutex = Mutex.new
+      @cache = {}
+    end
+
+    # Return the contents of the file at the given path.
+    # The file is read in the 'rb' mode and UTF-8 encoding.
+    #
+    # @param (see FileReader#read_file)
+    # @return (see FileReader#read_file)
+    # @note This method is cached, it will only access the filesystem on the first invocation.
+    def read_file(path)
+      absolute_path = File.expand_path(path)
+      @cache[absolute_path] || @mutex.synchronize { @cache[absolute_path] ||= super }
+    end
+  end
+end

data/lib/i18n/processes/scanners/files/file_finder.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module I18n::Processes::Scanners::Files
+  # Finds the files in the specified search paths with support for exclusion / inclusion patterns.
+  #
+  # @since 0.9.0
+  class FileFinder
+    include I18n::Processes::Logging
+
+    # @param paths [Array<String>] {Find.find}-compatible paths to traverse,
+    #     absolute or relative to the working directory.
+    # @param only [Array<String>, nil] {File.fnmatch}-compatible patterns files to include.
+    #     Files not matching any of the inclusion patterns will be excluded.
+    # @param exclude [Arry<String>] {File.fnmatch}-compatible patterns of files to exclude.
+    #     Files matching any of the exclusion patterns will be excluded even if they match an inclusion pattern.
+    def initialize(paths: ['.'], only: nil, exclude: [])
+      fail 'paths argument is required' if paths.nil?
+      @paths   = paths
+      @include = only
+      @exclude = exclude || []
+    end
+
+    # Traverse the paths and yield the matching ones.
+    #
+    # @yield [path]
+    # @yieldparam path [String] the path of the found file.
+    # @return [Array<of block results>]
+    def traverse_files
+      find_files.map { |path| yield path }
+    end
+
+    # @return [Array<String>] found files
+    def find_files # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      results = []
+      paths = @paths.select { |p| File.exist?(p) }
+      log_warn "None of the search.paths exist #{@paths.inspect}" if paths.empty?
+      Find.find(*paths) do |path|
+        is_dir   = File.directory?(path)
+        hidden   = File.basename(path).start_with?('.') && !%w[. ./].include?(path)
+        not_incl = @include && !path_fnmatch_any?(path, @include)
+        excl     = path_fnmatch_any?(path, @exclude)
+        if is_dir || hidden || not_incl || excl
+          Find.prune if is_dir && (hidden || excl)
+        else
+          results << path
+        end
+      end
+      results
+    end
+
+    private
+
+    # @param path [String]
+    # @param globs [Array<String>]
+    # @return [Boolean]
+    def path_fnmatch_any?(path, globs)
+      globs.any? { |glob| File.fnmatch(glob, path) }
+    end
+  end
+end

data/lib/i18n/processes/scanners/files/file_reader.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module I18n::Processes::Scanners::Files
+  # Reads the files in 'rb' mode and UTF-8 encoding.
+  #
+  # @since 0.9.0
+  class FileReader
+    # Return the contents of the file at the given path.
+    # The file is read in the 'rb' mode and UTF-8 encoding.
+    #
+    # @param path [String] Path to the file, absolute or relative to the working directory.
+    # @return [String] file contents
+    def read_file(path)
+      result = nil
+      File.open(path, 'rb', encoding: 'UTF-8') { |f| result = f.read }
+      result
+    end
+  end
+end

data/lib/i18n/processes/scanners/occurrence_from_position.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module I18n::Processes
+  module Scanners
+    module OccurrenceFromPosition
+      # Given a path to a file, its contents and a position in the file,
+      # return a {Results::Occurrence} at the position until the end of the line.
+      #
+      # @param path [String]
+      # @param contents [String] contents of the file at the path.
+      # @param position [Integer] position just before the beginning of the match.
+      # @return [Results::Occurrence]
+      def occurrence_from_position(path, contents, position, raw_key: nil)
+        line_begin = contents.rindex(/^/, position - 1)
+        line_end   = contents.index(/.(?=\r?\n|$)/, position)
+        Results::Occurrence.new(
+          path:     path,
+          pos:      position,
+          line_num: contents[0..position].count("\n") + 1,
+          line_pos: position - line_begin + 1,
+          line:     contents[line_begin..line_end],
+          raw_key: raw_key
+        )
+      end
+    end
+  end
+end

data/lib/i18n/processes/scanners/pattern_mapper.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/scanners/file_scanner'
+require 'i18n/processes/scanners/relative_keys'
+require 'i18n/processes/scanners/occurrence_from_position'
+require 'i18n/processes/scanners/ruby_key_literals'
+
+module I18n::Processes::Scanners
+  # Maps the provided patterns to keys.
+  class PatternMapper < FileScanner
+    include I18n::Processes::Scanners::RelativeKeys
+    include I18n::Processes::Scanners::OccurrenceFromPosition
+    include I18n::Processes::Scanners::RubyKeyLiterals
+
+    # @param patterns [Array<[String, String]> the list of pattern-key pairs
+    #   the patterns follow the regular expression syntax, with a syntax addition for matching
+    #   string/symbol literals: you can include %{key} in the pattern, and it will be converted to
+    #   a named capture group, capturing ruby strings and symbols, that can then be used in the key:
+    #
+    #       patterns: [['Spree\.t[( ]\s*%{key}', 'spree.%{key}']]
+    #
+    #   All of the named capture groups are interpolated into the key with %{group_name} interpolations.
+    #
+    def initialize(config:, **args)
+      super
+      @patterns = configure_patterns(config[:patterns] || [])
+    end
+
+    protected
+
+    # @return [Array<[absolute key, Results::Occurrence]>]
+    def scan_file(path)
+      text = read_file(path)
+      @patterns.flat_map do |pattern, key|
+        result = []
+        text.scan(pattern) do |_|
+          match    = Regexp.last_match
+          matches  = Hash[match.names.map(&:to_sym).zip(match.captures)]
+          if matches.key?(:key)
+            matches[:key] = strip_literal(matches[:key])
+            next unless valid_key?(matches[:key])
+          end
+          result << [absolute_key(format(key, matches), path),
+                     occurrence_from_position(path, text, match.offset(0).first)]
+        end
+        result
+      end
+    end
+
+    private
+
+    KEY_GROUP = "(?<key>#{LITERAL_RE})"
+
+    def configure_patterns(patterns)
+      patterns.map do |(pattern, key)|
+        [pattern.is_a?(Regexp) ? pattern : Regexp.new(format(pattern, key: KEY_GROUP)), key]
+      end
+    end
+  end
+end