i18n-tasks 0.8.7 → 0.9.0.rc1

data/lib/i18n/tasks/scanners/files/caching_file_reader.rb

@@ -0,0 +1,27 @@
+require 'i18n/tasks/scanners/files/file_reader'
+module I18n::Tasks::Scanners::Files
+  # Reads the files in 'rb' mode and UTF-8 encoding.
+  # Wraps a {FileReader} and caches the results.
+  #
+  # @note This class is thread-safe. All methods are cached.
+  # @since 0.9.0
+  class CachingFileReader < FileReader
+
+    def initialize
+      super
+      @mutex = Mutex.new
+      @cache = {}
+    end
+
+    # Return the contents of the file at the given path.
+    # The file is read in the 'rb' mode and UTF-8 encoding.
+    #
+    # @param (see FileReader#read_file)
+    # @return (see FileReader#read_file)
+    # @note This method is cached, it will only access the filesystem on the first invocation.
+    def read_file(path)
+      absolute_path = File.expand_path(path)
+      @cache[absolute_path] || @mutex.synchronize { @cache[absolute_path] ||= super }
+    end
+  end
+end

data/lib/i18n/tasks/scanners/files/file_finder.rb

@@ -0,0 +1,61 @@
+module I18n::Tasks::Scanners::Files
+  # Finds the files in the specified search paths with support for exclusion / inclusion patterns.
+  #
+  # @since 0.9.0
+  class FileFinder
+    include I18n::Tasks::Logging
+
+    # @param paths [Array<String>] {Find.find}-compatible paths to traverse,
+    #     absolute or relative to the working directory.
+    # @param include [Array<String>, nil] {File.fnmatch}-compatible patterns files to include.
+    #     Files not matching any of the inclusion patterns will be excluded.
+    # @param exclude [Arry<String>] {File.fnmatch}-compatible patterns of files to exclude.
+    #     Files matching any of the exclusion patterns will be excluded even if they match an inclusion pattern.
+    def initialize(paths: ['.'], include: nil, exclude: [])
+      raise 'paths argument is required' if paths.nil?
+      @paths   = paths
+      @include = include
+      @exclude = exclude || []
+    end
+
+    # Traverse the paths and yield the matching ones.
+    #
+    # @yield [path]
+    # @yieldparam path [String] the path of the found file.
+    # @return [Array<of block results>]
+    def traverse_files
+      find_files.map { |path| yield path }
+    end
+
+    # @return [Array<String>] found files
+    def find_files
+      results = []
+      paths = @paths.select { |p| File.exist?(p) }
+      if paths.empty?
+        log_warn "None of the search.paths exist #{@paths.inspect}"
+      else
+        Find.find(*paths) do |path|
+          is_dir   = File.directory?(path)
+          hidden   = File.basename(path).start_with?('.') && !%w(. ./).include?(path)
+          not_incl = @include && !path_fnmatch_any?(path, @include)
+          excl     = path_fnmatch_any?(path, @exclude)
+          if is_dir || hidden || not_incl || excl
+            Find.prune if is_dir && (hidden || excl)
+          else
+            results << path
+          end
+        end
+      end
+      results
+    end
+
+    private
+
+    # @param path [String]
+    # @param globs [Array<String>]
+    # @return [Boolean]
+    def path_fnmatch_any?(path, globs)
+      globs.any? { |glob| File.fnmatch(glob, path) }
+    end
+  end
+end

data/lib/i18n/tasks/scanners/files/file_reader.rb

@@ -0,0 +1,18 @@
+module I18n::Tasks::Scanners::Files
+  # Reads the files in 'rb' mode and UTF-8 encoding.
+  #
+  # @since 0.9.0
+  class FileReader
+
+    # Return the contents of the file at the given path.
+    # The file is read in the 'rb' mode and UTF-8 encoding.
+    #
+    # @param path [String] Path to the file, absolute or relative to the working directory.
+    # @return [String] file contents
+    def read_file(path)
+      result = nil
+      File.open(path, 'rb', encoding: 'UTF-8') { |f| result = f.read }
+      result
+    end
+  end
+end

data/lib/i18n/tasks/scanners/key_occurrences.rb

@@ -0,0 +1,35 @@
+require 'i18n/tasks/scanners/occurence'
+
+module I18n::Tasks::Scanners
+  # A scanned key and all its occurrences.
+  #
+  # @note This is a value type. Equality and hash code are determined from the attributes.
+  class KeyOccurrences
+    # @return [String] the key.
+    attr_reader :key
+
+    # @return [Array<Occurrence>] the key's occurrences.
+    attr_reader :occurrences
+
+    def initialize(key:, occurrences:)
+      @key         = key
+      @occurrences = occurrences
+    end
+
+    def ==(other)
+      other.key == @key && other.occurrences == @occurrences
+    end
+
+    def eql?(other)
+      self == other
+    end
+
+    def hash
+      [@key, @occurrences].hash
+    end
+
+    def inspect
+      "KeyOccurrences(#{key.inspect}, [#{occurrences.map(&:inspect).join(', ')}])"
+    end
+  end
+end

data/lib/i18n/tasks/scanners/occurence.rb

@@ -0,0 +1,50 @@
+module I18n::Tasks::Scanners
+  # The occurrence of some key in a file.
+  #
+  # @note This is a value type. Equality and hash code are determined from the attributes.
+  class Occurrence
+    # @return [String] source path relative to the current working directory.
+    attr_reader :path
+
+    # @return [Fixnum] count of characters in the file before the occurrence.
+    attr_reader :pos
+
+    # @return [Fixnum] line number of the occurrence, counting from 1.
+    attr_reader :line_num
+
+    # @return [Fixnum] position of the start of the occurrence in the line, counting from 1.
+    attr_reader :line_pos
+
+    # @return [String] the line of the occurrence, excluding the last LF or CRLF.
+    attr_reader :line
+
+    # @param path     [String]
+    # @param pos      [Fixnum]
+    # @param line_num [Fixnum]
+    # @param line_pos [Fixnum]
+    # @param line     [String]
+    def initialize(path:, pos:, line_num:, line_pos:, line:)
+      @path     = path
+      @pos      = pos
+      @line_num = line_num
+      @line_pos = line_pos
+      @line     = line
+    end
+
+    def inspect
+      "Occurrence(#{@path}:#{@line_num}:#{@line_pos}:(#{@pos})"
+    end
+
+    def ==(other)
+      other.path == @path && other.pos == @pos && other.line_num == @line_num && other.line == @line
+    end
+
+    def eql?(other)
+      self == other
+    end
+
+    def hash
+      [@path, @pos, @line_num, @line_pos, @line].hash
+    end
+  end
+end

data/lib/i18n/tasks/scanners/pattern_scanner.rb

@@ -1,62 +1,117 @@
-# coding: utf-8
-require 'i18n/tasks/scanners/base_scanner'
+require 'i18n/tasks/scanners/scanner'
+require 'i18n/tasks/scanners/relative_keys'
 
 module I18n::Tasks::Scanners
-  # Scans for I18n.t usages
-  #
-  class PatternScanner < BaseScanner
+  # Scan for I18n.t usages using a simple regular expression.
+  class PatternScanner < Scanner
+    include RelativeKeys
+
+    attr_reader :config
+
+    def initialize(
+        config: {},
+        file_finder_provider: Files::CachingFileFinderProvider.new,
+        file_reader: Files::CachingFileReader.new)
+      @config      = config
+      @file_reader = file_reader
+
+      @file_finder      = file_finder_provider.get(**config.slice(:paths, :include, :exclude))
+      @pattern          = config[:pattern].present? ? Regexp.new(config[:pattern]) : default_pattern
+      @ignore_lines_res = (config[:ignore_lines] || []).inject({}) { |h, (ext, re)| h.update(ext => Regexp.new(re)) }
+    end
+
+    # @return (see Scanner#keys)
+    def keys
+      (@file_finder.traverse_files { |path|
+        scan_file(path)
+      }.reduce(:+) || []).group_by(&:first).map { |key, keys_occurrences|
+        KeyOccurrences.new(key: key, occurrences: keys_occurrences.map(&:second))
+      }
+    end
+
+    protected
+
     # Extract i18n keys from file based on the pattern which must capture the key literal.
-    # @return [Array<Key>] keys found in file
-    def scan_file(path, opts = {})
+    # @return [Array<[key, occurrence]>] each occurrence found in the file
+    def scan_file(path)
       keys = []
-      strict = !!opts[:strict]
-      text = opts[:text] || read_file(path)
-      text.scan(pattern) do |match|
-        src_pos = Regexp.last_match.offset(0).first
+      text = @file_reader.read_file(path)
+      text.scan(@pattern) do |match|
+        src_pos  = Regexp.last_match.offset(0).first
         location = src_location(path, text, src_pos)
-        next if exclude_line?(location[:line], path)
+        next if exclude_line?(location.line, path)
         key = match_to_key(match, path, location)
         next unless key
         key = key + ':' if key.end_with?('.')
-        next unless valid_key?(key, strict)
-        keys << [key, data: location]
+        next unless valid_key?(key)
+        keys << [key, location]
       end
       keys
     rescue Exception => e
-      raise ::I18n::Tasks::CommandError.new("Error scanning #{path}: #{e.message}")
-    end
-
-    def default_pattern
-      # capture only the first argument
-      /
-      #{translate_call_re} [\( ] \s* (?# fn call begin )
-      (#{literal_re})                (?# capture the first argument)
-      /x
+      raise ::I18n::Tasks::CommandError.new(e, "Error scanning #{path}: #{e.message}")
     end
 
-    protected
-
-    # Given
     # @param [MatchData] match
     # @param [String] path
     # @return [String] full absolute key name
     def match_to_key(match, path, location)
-      key = strip_literal(match[0])
-      absolute_key(key, path, location)
+      absolute_key(strip_literal(match[0]), path, location)
+    end
+
+    def exclude_line?(line, path)
+      re = @ignore_lines_res[File.extname(path)[1..-1]]
+      re && re =~ line
     end
 
     def absolute_key(key, path, location)
       if key.start_with?('.')
         if controller_file?(path) || mailer_file?(path)
-          absolutize_key(key, path, relative_roots, closest_method(location))
+          absolutize_key(key, path, config[:relative_roots], closest_method(location))
         else
-          absolutize_key(key, path)
+          absolutize_key(key, path, config[:relative_roots])
         end
       else
         key
       end
     end
 
+    # @param path [String]
+    # @param text [String] contents of the file at the path.
+    # @param src_pos [Fixnum] position just before the beginning of the match.
+    # @return [Occurrence]
+    def src_location(path, text, src_pos)
+      line_begin = text.rindex(/^/, src_pos - 1)
+      line_end   = text.index(/.(?=\r?\n|$)/, src_pos)
+      Occurrence.new(
+          path:     path,
+          pos:      src_pos,
+          line_num: text[0..src_pos].count("\n") + 1,
+          line_pos: src_pos - line_begin + 1,
+          line:     text[line_begin..line_end])
+    end
+
+    # remove the leading colon and unwrap quotes from the key match
+    # @param literal [String] e.g: "key", 'key', or :key.
+    # @return [String] key
+    def strip_literal(literal)
+      key = literal
+      key = key[1..-1] if ':' == key[0]
+      key = key[1..-2] if %w(' ").include?(key[0])
+      key
+    end
+
+    VALID_KEY_CHARS     = /(?:[[:word:]]|[-.?!;À-ž])/
+    VALID_KEY_RE_STRICT = /^#{VALID_KEY_CHARS}+$/
+    VALID_KEY_RE        = /^(#{VALID_KEY_CHARS}|[:\#{@}\[\]])+$/
+
+    def valid_key?(key)
+      if @config[:strict]
+        key =~ VALID_KEY_RE_STRICT && !key.end_with?('.')
+      else
+        key =~ VALID_KEY_RE
+      end
+    end
+
     def controller_file?(path)
       /controllers/.match(path)
     end
@@ -65,14 +120,10 @@ module I18n::Tasks::Scanners
       /mailers/.match(path)
     end
 
-    def closest_method(location)
-      method = File.readlines(location[:src_path], encoding: 'UTF-8').first(location[:line_num] - 1).reverse_each.find { |x| x=~ /\bdef\b/ }
-      method &&= method.strip.sub(/^def\s*/, '').sub(/[\(\s;].*$/, '')
-      method
-    end
-
-    def pattern
-      @pattern ||= config[:pattern].present? ? Regexp.new(config[:pattern]) : default_pattern
+    def closest_method(occurrence)
+      method = File.readlines(occurrence.path, encoding: 'UTF-8').
+          first(occurrence.line_num - 1).reverse_each.find { |x| x =~ /\bdef\b/ }
+      method && method.strip.sub(/^def\s*/, '').sub(/[\(\s;].*$/, '')
     end
 
     def translate_call_re
@@ -85,5 +136,13 @@ module I18n::Tasks::Scanners
     def literal_re
       /:?".+?"|:?'.+?'|:\w+/
     end
+
+    def default_pattern
+      # capture only the first argument
+      /
+      #{translate_call_re} [\( ] \s* (?# fn call begin )
+      (#{literal_re})                (?# capture the first argument)
+      /x
+    end
   end
 end

data/lib/i18n/tasks/scanners/pattern_with_scope_scanner.rb

@@ -1,4 +1,3 @@
-# coding: utf-8
 require 'i18n/tasks/scanners/pattern_scanner'
 
 module I18n::Tasks::Scanners
@@ -7,6 +6,8 @@ module I18n::Tasks::Scanners
   # Caveat: scope is only detected when it is the first argument
   class PatternWithScopeScanner < PatternScanner
 
+    protected
+
     def default_pattern
       # capture the first argument and scope argument if present
       /#{super}
@@ -14,8 +15,6 @@ module I18n::Tasks::Scanners
       /x
     end
 
-    protected
-
     # Given
     # @param [MatchData] match
     # @param [String] path

data/lib/i18n/tasks/scanners/relative_keys.rb

@@ -1,4 +1,3 @@
-# coding: utf-8
 module I18n
   module Tasks
     module Scanners
@@ -7,13 +6,13 @@ module I18n
         # @param path [String] path to the file containing the key
         # @return [String] absolute version of the key
         def absolutize_key(key, path, roots = relative_roots, closest_method = "")
+          fail 'roots argument is required' if roots.nil?
           normalized_path = File.expand_path(path)
           path_root(normalized_path, roots) or
-            raise CommandError.new(
+            fail CommandError.new(
               "Error scanning #{normalized_path}: cannot resolve relative key
               \"#{key}\".\nSet search.relative_roots in config/i18n-tasks.yml
-              (currently #{relative_roots.inspect})"
-            )
+              (currently #{roots.inspect})")
 
           prefix_key_based_on_path(key, normalized_path, roots, closest_method: closest_method)
         end

data/lib/i18n/tasks/scanners/scanner.rb

@@ -0,0 +1,15 @@
+require 'i18n/tasks/scanners/key_occurrences'
+
+module I18n::Tasks::Scanners
+  # Describes the API of a scanner.
+  #
+  # @abstract
+  # @since 0.9.0
+  class Scanner
+    # @abstract
+    # @return [Array<KeyOccurrences>] the keys found by this scanner and their occurrences.
+    def keys
+      raise 'Unimplemented'
+    end
+  end
+end

data/lib/i18n/tasks/scanners/scanner_multiplexer.rb

@@ -0,0 +1,43 @@
+require 'i18n/tasks/scanners/scanner'
+
+module I18n::Tasks::Scanners
+  # Run multiple {Scanner Scanners} and merge their results.
+  # @note The scanners are run concurrently. A thread is spawned per each scanner.
+  # @since 0.9.0
+  class ScannerMultiplexer < Scanner
+    # @param scanners [Array<Scanner>]
+    def initialize(scanners:)
+      @scanners = scanners
+    end
+
+    # Collect the results of all the scanners. Occurrences of a key from multiple scanners are merged.
+    #
+    # @note The scanners are run concurrently. A thread is spawned per each scanner.
+    # @return (see Scanner#keys)
+    def keys
+      collect_results.inject({}) { |results_by_key, key_occurences|
+        key_occurences.each do |key_occurrence|
+          (results_by_key[key_occurrence.key] ||= []) << key_occurrence.occurrences
+        end
+        results_by_key
+      }.map { |key, all_occurrences|
+        occurrences = all_occurrences.flatten(1)
+        occurrences.sort_by!(&:path)
+        occurrences.uniq!
+        KeyOccurrences.new(key: key, occurrences: occurrences)
+      }
+    end
+
+    private
+
+    # @return Array<Array<KeyOccurrences>>
+    def collect_results
+      return [@scanners[0].keys] if @scanners.length == 1
+      Array.new(@scanners.length).tap do |results|
+        @scanners.map.with_index { |scanner, i|
+          Thread.start { results[i] = scanner.keys }
+        }.each(&:join)
+      end
+    end
+  end
+end