i18n-tasks 0.8.7 → 0.9.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +6 -5
  3. data/CHANGES.md +3 -3
  4. data/Gemfile +1 -1
  5. data/README.md +4 -4
  6. data/bin/i18n-tasks +0 -1
  7. data/config/locales/en.yml +103 -102
  8. data/config/locales/ru.yml +1 -1
  9. data/i18n-tasks.gemspec +1 -2
  10. data/lib/i18n/tasks.rb +0 -1
  11. data/lib/i18n/tasks/base_task.rb +0 -1
  12. data/lib/i18n/tasks/cli.rb +1 -1
  13. data/lib/i18n/tasks/command/commander.rb +0 -1
  14. data/lib/i18n/tasks/command/commands/missing.rb +3 -15
  15. data/lib/i18n/tasks/command/commands/usages.rb +5 -6
  16. data/lib/i18n/tasks/command/option_parsers/locale.rb +1 -8
  17. data/lib/i18n/tasks/command_error.rb +5 -1
  18. data/lib/i18n/tasks/commands.rb +0 -1
  19. data/lib/i18n/tasks/configuration.rb +1 -9
  20. data/lib/i18n/tasks/console_context.rb +2 -2
  21. data/lib/i18n/tasks/data.rb +0 -1
  22. data/lib/i18n/tasks/data/adapter/json_adapter.rb +0 -1
  23. data/lib/i18n/tasks/data/adapter/yaml_adapter.rb +0 -1
  24. data/lib/i18n/tasks/data/file_formats.rb +0 -1
  25. data/lib/i18n/tasks/data/file_system.rb +0 -1
  26. data/lib/i18n/tasks/data/file_system_base.rb +0 -1
  27. data/lib/i18n/tasks/data/router/conservative_router.rb +0 -1
  28. data/lib/i18n/tasks/data/router/pattern_router.rb +0 -1
  29. data/lib/i18n/tasks/data/tree/node.rb +6 -7
  30. data/lib/i18n/tasks/data/tree/nodes.rb +0 -1
  31. data/lib/i18n/tasks/data/tree/siblings.rb +29 -10
  32. data/lib/i18n/tasks/data/tree/traversal.rb +0 -3
  33. data/lib/i18n/tasks/google_translation.rb +0 -1
  34. data/lib/i18n/tasks/ignore_keys.rb +0 -1
  35. data/lib/i18n/tasks/key_pattern_matching.rb +0 -1
  36. data/lib/i18n/tasks/logging.rb +0 -1
  37. data/lib/i18n/tasks/missing_keys.rb +0 -1
  38. data/lib/i18n/tasks/plural_keys.rb +0 -1
  39. data/lib/i18n/tasks/reports/base.rb +1 -2
  40. data/lib/i18n/tasks/reports/spreadsheet.rb +0 -1
  41. data/lib/i18n/tasks/reports/terminal.rb +41 -17
  42. data/lib/i18n/tasks/scanners/files/caching_file_finder.rb +32 -0
  43. data/lib/i18n/tasks/scanners/files/caching_file_finder_provider.rb +24 -0
  44. data/lib/i18n/tasks/scanners/files/caching_file_reader.rb +27 -0
  45. data/lib/i18n/tasks/scanners/files/file_finder.rb +61 -0
  46. data/lib/i18n/tasks/scanners/files/file_reader.rb +18 -0
  47. data/lib/i18n/tasks/scanners/key_occurrences.rb +35 -0
  48. data/lib/i18n/tasks/scanners/occurence.rb +50 -0
  49. data/lib/i18n/tasks/scanners/pattern_scanner.rb +97 -38
  50. data/lib/i18n/tasks/scanners/pattern_with_scope_scanner.rb +2 -3
  51. data/lib/i18n/tasks/scanners/relative_keys.rb +3 -4
  52. data/lib/i18n/tasks/scanners/scanner.rb +15 -0
  53. data/lib/i18n/tasks/scanners/scanner_multiplexer.rb +43 -0
  54. data/lib/i18n/tasks/unused_keys.rb +4 -5
  55. data/lib/i18n/tasks/used_keys.rb +76 -23
  56. data/lib/i18n/tasks/version.rb +1 -2
  57. data/spec/conservative_router_spec.rb +0 -1
  58. data/spec/file_system_data_spec.rb +0 -1
  59. data/spec/fixtures/app/controllers/events_controller.rb +1 -2
  60. data/spec/google_translate_spec.rb +0 -1
  61. data/spec/i18n_tasks_spec.rb +4 -15
  62. data/spec/key_pattern_matching_spec.rb +0 -1
  63. data/spec/locale_tree/siblings_spec.rb +0 -1
  64. data/spec/pattern_scanner_spec.rb +34 -36
  65. data/spec/plural_keys_spec.rb +0 -1
  66. data/spec/readme_spec.rb +0 -1
  67. data/spec/relative_keys_spec.rb +15 -10
  68. data/spec/scanners/files/caching_file_finder_provider_spec.rb +18 -0
  69. data/spec/scanners/files/caching_file_finder_spec.rb +39 -0
  70. data/spec/scanners/files/caching_file_reader_spec.rb +18 -0
  71. data/spec/scanners/files/file_finder_spec.rb +52 -0
  72. data/spec/scanners/files/file_reader_spec.rb +15 -0
  73. data/spec/scanners/scanner_multiplexer_spec.rb +26 -0
  74. data/spec/spec_helper.rb +1 -1
  75. data/spec/support/capture_std.rb +0 -1
  76. data/spec/support/fixtures.rb +0 -1
  77. data/spec/support/i18n_tasks_output_matcher.rb +0 -1
  78. data/spec/support/key_pattern_matcher.rb +0 -1
  79. data/spec/support/keys_and_occurrences.rb +27 -0
  80. data/spec/support/test_codebase.rb +0 -1
  81. data/spec/support/trees.rb +1 -7
  82. data/spec/used_keys_spec.rb +15 -16
  83. data/templates/config/i18n-tasks.yml +9 -2
  84. metadata +29 -9
  85. data/lib/i18n/tasks/scanners/base_scanner.rb +0 -149
  86. data/spec/commands/missing_commands_spec.rb +0 -23
@@ -0,0 +1,27 @@
1
+ require 'i18n/tasks/scanners/files/file_reader'
2
+ module I18n::Tasks::Scanners::Files
3
+ # Reads the files in 'rb' mode and UTF-8 encoding.
4
+ # Wraps a {FileReader} and caches the results.
5
+ #
6
+ # @note This class is thread-safe. All methods are cached.
7
+ # @since 0.9.0
8
+ class CachingFileReader < FileReader
9
+
10
+ def initialize
11
+ super
12
+ @mutex = Mutex.new
13
+ @cache = {}
14
+ end
15
+
16
+ # Return the contents of the file at the given path.
17
+ # The file is read in the 'rb' mode and UTF-8 encoding.
18
+ #
19
+ # @param (see FileReader#read_file)
20
+ # @return (see FileReader#read_file)
21
+ # @note This method is cached, it will only access the filesystem on the first invocation.
22
+ def read_file(path)
23
+ absolute_path = File.expand_path(path)
24
+ @cache[absolute_path] || @mutex.synchronize { @cache[absolute_path] ||= super }
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,61 @@
1
+ module I18n::Tasks::Scanners::Files
2
+ # Finds the files in the specified search paths with support for exclusion / inclusion patterns.
3
+ #
4
+ # @since 0.9.0
5
+ class FileFinder
6
+ include I18n::Tasks::Logging
7
+
8
+ # @param paths [Array<String>] {Find.find}-compatible paths to traverse,
9
+ # absolute or relative to the working directory.
10
+ # @param include [Array<String>, nil] {File.fnmatch}-compatible patterns files to include.
11
+ # Files not matching any of the inclusion patterns will be excluded.
12
+ # @param exclude [Arry<String>] {File.fnmatch}-compatible patterns of files to exclude.
13
+ # Files matching any of the exclusion patterns will be excluded even if they match an inclusion pattern.
14
+ def initialize(paths: ['.'], include: nil, exclude: [])
15
+ raise 'paths argument is required' if paths.nil?
16
+ @paths = paths
17
+ @include = include
18
+ @exclude = exclude || []
19
+ end
20
+
21
+ # Traverse the paths and yield the matching ones.
22
+ #
23
+ # @yield [path]
24
+ # @yieldparam path [String] the path of the found file.
25
+ # @return [Array<of block results>]
26
+ def traverse_files
27
+ find_files.map { |path| yield path }
28
+ end
29
+
30
+ # @return [Array<String>] found files
31
+ def find_files
32
+ results = []
33
+ paths = @paths.select { |p| File.exist?(p) }
34
+ if paths.empty?
35
+ log_warn "None of the search.paths exist #{@paths.inspect}"
36
+ else
37
+ Find.find(*paths) do |path|
38
+ is_dir = File.directory?(path)
39
+ hidden = File.basename(path).start_with?('.') && !%w(. ./).include?(path)
40
+ not_incl = @include && !path_fnmatch_any?(path, @include)
41
+ excl = path_fnmatch_any?(path, @exclude)
42
+ if is_dir || hidden || not_incl || excl
43
+ Find.prune if is_dir && (hidden || excl)
44
+ else
45
+ results << path
46
+ end
47
+ end
48
+ end
49
+ results
50
+ end
51
+
52
+ private
53
+
54
+ # @param path [String]
55
+ # @param globs [Array<String>]
56
+ # @return [Boolean]
57
+ def path_fnmatch_any?(path, globs)
58
+ globs.any? { |glob| File.fnmatch(glob, path) }
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,18 @@
1
+ module I18n::Tasks::Scanners::Files
2
+ # Reads the files in 'rb' mode and UTF-8 encoding.
3
+ #
4
+ # @since 0.9.0
5
+ class FileReader
6
+
7
+ # Return the contents of the file at the given path.
8
+ # The file is read in the 'rb' mode and UTF-8 encoding.
9
+ #
10
+ # @param path [String] Path to the file, absolute or relative to the working directory.
11
+ # @return [String] file contents
12
+ def read_file(path)
13
+ result = nil
14
+ File.open(path, 'rb', encoding: 'UTF-8') { |f| result = f.read }
15
+ result
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,35 @@
1
+ require 'i18n/tasks/scanners/occurence'
2
+
3
+ module I18n::Tasks::Scanners
4
+ # A scanned key and all its occurrences.
5
+ #
6
+ # @note This is a value type. Equality and hash code are determined from the attributes.
7
+ class KeyOccurrences
8
+ # @return [String] the key.
9
+ attr_reader :key
10
+
11
+ # @return [Array<Occurrence>] the key's occurrences.
12
+ attr_reader :occurrences
13
+
14
+ def initialize(key:, occurrences:)
15
+ @key = key
16
+ @occurrences = occurrences
17
+ end
18
+
19
+ def ==(other)
20
+ other.key == @key && other.occurrences == @occurrences
21
+ end
22
+
23
+ def eql?(other)
24
+ self == other
25
+ end
26
+
27
+ def hash
28
+ [@key, @occurrences].hash
29
+ end
30
+
31
+ def inspect
32
+ "KeyOccurrences(#{key.inspect}, [#{occurrences.map(&:inspect).join(', ')}])"
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,50 @@
1
+ module I18n::Tasks::Scanners
2
+ # The occurrence of some key in a file.
3
+ #
4
+ # @note This is a value type. Equality and hash code are determined from the attributes.
5
+ class Occurrence
6
+ # @return [String] source path relative to the current working directory.
7
+ attr_reader :path
8
+
9
+ # @return [Fixnum] count of characters in the file before the occurrence.
10
+ attr_reader :pos
11
+
12
+ # @return [Fixnum] line number of the occurrence, counting from 1.
13
+ attr_reader :line_num
14
+
15
+ # @return [Fixnum] position of the start of the occurrence in the line, counting from 1.
16
+ attr_reader :line_pos
17
+
18
+ # @return [String] the line of the occurrence, excluding the last LF or CRLF.
19
+ attr_reader :line
20
+
21
+ # @param path [String]
22
+ # @param pos [Fixnum]
23
+ # @param line_num [Fixnum]
24
+ # @param line_pos [Fixnum]
25
+ # @param line [String]
26
+ def initialize(path:, pos:, line_num:, line_pos:, line:)
27
+ @path = path
28
+ @pos = pos
29
+ @line_num = line_num
30
+ @line_pos = line_pos
31
+ @line = line
32
+ end
33
+
34
+ def inspect
35
+ "Occurrence(#{@path}:#{@line_num}:#{@line_pos}:(#{@pos})"
36
+ end
37
+
38
+ def ==(other)
39
+ other.path == @path && other.pos == @pos && other.line_num == @line_num && other.line == @line
40
+ end
41
+
42
+ def eql?(other)
43
+ self == other
44
+ end
45
+
46
+ def hash
47
+ [@path, @pos, @line_num, @line_pos, @line].hash
48
+ end
49
+ end
50
+ end
@@ -1,62 +1,117 @@
1
- # coding: utf-8
2
- require 'i18n/tasks/scanners/base_scanner'
1
+ require 'i18n/tasks/scanners/scanner'
2
+ require 'i18n/tasks/scanners/relative_keys'
3
3
 
4
4
  module I18n::Tasks::Scanners
5
- # Scans for I18n.t usages
6
- #
7
- class PatternScanner < BaseScanner
5
+ # Scan for I18n.t usages using a simple regular expression.
6
+ class PatternScanner < Scanner
7
+ include RelativeKeys
8
+
9
+ attr_reader :config
10
+
11
+ def initialize(
12
+ config: {},
13
+ file_finder_provider: Files::CachingFileFinderProvider.new,
14
+ file_reader: Files::CachingFileReader.new)
15
+ @config = config
16
+ @file_reader = file_reader
17
+
18
+ @file_finder = file_finder_provider.get(**config.slice(:paths, :include, :exclude))
19
+ @pattern = config[:pattern].present? ? Regexp.new(config[:pattern]) : default_pattern
20
+ @ignore_lines_res = (config[:ignore_lines] || []).inject({}) { |h, (ext, re)| h.update(ext => Regexp.new(re)) }
21
+ end
22
+
23
+ # @return (see Scanner#keys)
24
+ def keys
25
+ (@file_finder.traverse_files { |path|
26
+ scan_file(path)
27
+ }.reduce(:+) || []).group_by(&:first).map { |key, keys_occurrences|
28
+ KeyOccurrences.new(key: key, occurrences: keys_occurrences.map(&:second))
29
+ }
30
+ end
31
+
32
+ protected
33
+
8
34
  # Extract i18n keys from file based on the pattern which must capture the key literal.
9
- # @return [Array<Key>] keys found in file
10
- def scan_file(path, opts = {})
35
+ # @return [Array<[key, occurrence]>] each occurrence found in the file
36
+ def scan_file(path)
11
37
  keys = []
12
- strict = !!opts[:strict]
13
- text = opts[:text] || read_file(path)
14
- text.scan(pattern) do |match|
15
- src_pos = Regexp.last_match.offset(0).first
38
+ text = @file_reader.read_file(path)
39
+ text.scan(@pattern) do |match|
40
+ src_pos = Regexp.last_match.offset(0).first
16
41
  location = src_location(path, text, src_pos)
17
- next if exclude_line?(location[:line], path)
42
+ next if exclude_line?(location.line, path)
18
43
  key = match_to_key(match, path, location)
19
44
  next unless key
20
45
  key = key + ':' if key.end_with?('.')
21
- next unless valid_key?(key, strict)
22
- keys << [key, data: location]
46
+ next unless valid_key?(key)
47
+ keys << [key, location]
23
48
  end
24
49
  keys
25
50
  rescue Exception => e
26
- raise ::I18n::Tasks::CommandError.new("Error scanning #{path}: #{e.message}")
27
- end
28
-
29
- def default_pattern
30
- # capture only the first argument
31
- /
32
- #{translate_call_re} [\( ] \s* (?# fn call begin )
33
- (#{literal_re}) (?# capture the first argument)
34
- /x
51
+ raise ::I18n::Tasks::CommandError.new(e, "Error scanning #{path}: #{e.message}")
35
52
  end
36
53
 
37
- protected
38
-
39
- # Given
40
54
  # @param [MatchData] match
41
55
  # @param [String] path
42
56
  # @return [String] full absolute key name
43
57
  def match_to_key(match, path, location)
44
- key = strip_literal(match[0])
45
- absolute_key(key, path, location)
58
+ absolute_key(strip_literal(match[0]), path, location)
59
+ end
60
+
61
+ def exclude_line?(line, path)
62
+ re = @ignore_lines_res[File.extname(path)[1..-1]]
63
+ re && re =~ line
46
64
  end
47
65
 
48
66
  def absolute_key(key, path, location)
49
67
  if key.start_with?('.')
50
68
  if controller_file?(path) || mailer_file?(path)
51
- absolutize_key(key, path, relative_roots, closest_method(location))
69
+ absolutize_key(key, path, config[:relative_roots], closest_method(location))
52
70
  else
53
- absolutize_key(key, path)
71
+ absolutize_key(key, path, config[:relative_roots])
54
72
  end
55
73
  else
56
74
  key
57
75
  end
58
76
  end
59
77
 
78
+ # @param path [String]
79
+ # @param text [String] contents of the file at the path.
80
+ # @param src_pos [Fixnum] position just before the beginning of the match.
81
+ # @return [Occurrence]
82
+ def src_location(path, text, src_pos)
83
+ line_begin = text.rindex(/^/, src_pos - 1)
84
+ line_end = text.index(/.(?=\r?\n|$)/, src_pos)
85
+ Occurrence.new(
86
+ path: path,
87
+ pos: src_pos,
88
+ line_num: text[0..src_pos].count("\n") + 1,
89
+ line_pos: src_pos - line_begin + 1,
90
+ line: text[line_begin..line_end])
91
+ end
92
+
93
+ # remove the leading colon and unwrap quotes from the key match
94
+ # @param literal [String] e.g: "key", 'key', or :key.
95
+ # @return [String] key
96
+ def strip_literal(literal)
97
+ key = literal
98
+ key = key[1..-1] if ':' == key[0]
99
+ key = key[1..-2] if %w(' ").include?(key[0])
100
+ key
101
+ end
102
+
103
+ VALID_KEY_CHARS = /(?:[[:word:]]|[-.?!;À-ž])/
104
+ VALID_KEY_RE_STRICT = /^#{VALID_KEY_CHARS}+$/
105
+ VALID_KEY_RE = /^(#{VALID_KEY_CHARS}|[:\#{@}\[\]])+$/
106
+
107
+ def valid_key?(key)
108
+ if @config[:strict]
109
+ key =~ VALID_KEY_RE_STRICT && !key.end_with?('.')
110
+ else
111
+ key =~ VALID_KEY_RE
112
+ end
113
+ end
114
+
60
115
  def controller_file?(path)
61
116
  /controllers/.match(path)
62
117
  end
@@ -65,14 +120,10 @@ module I18n::Tasks::Scanners
65
120
  /mailers/.match(path)
66
121
  end
67
122
 
68
- def closest_method(location)
69
- method = File.readlines(location[:src_path], encoding: 'UTF-8').first(location[:line_num] - 1).reverse_each.find { |x| x=~ /\bdef\b/ }
70
- method &&= method.strip.sub(/^def\s*/, '').sub(/[\(\s;].*$/, '')
71
- method
72
- end
73
-
74
- def pattern
75
- @pattern ||= config[:pattern].present? ? Regexp.new(config[:pattern]) : default_pattern
123
+ def closest_method(occurrence)
124
+ method = File.readlines(occurrence.path, encoding: 'UTF-8').
125
+ first(occurrence.line_num - 1).reverse_each.find { |x| x =~ /\bdef\b/ }
126
+ method && method.strip.sub(/^def\s*/, '').sub(/[\(\s;].*$/, '')
76
127
  end
77
128
 
78
129
  def translate_call_re
@@ -85,5 +136,13 @@ module I18n::Tasks::Scanners
85
136
  def literal_re
86
137
  /:?".+?"|:?'.+?'|:\w+/
87
138
  end
139
+
140
+ def default_pattern
141
+ # capture only the first argument
142
+ /
143
+ #{translate_call_re} [\( ] \s* (?# fn call begin )
144
+ (#{literal_re}) (?# capture the first argument)
145
+ /x
146
+ end
88
147
  end
89
148
  end
@@ -1,4 +1,3 @@
1
- # coding: utf-8
2
1
  require 'i18n/tasks/scanners/pattern_scanner'
3
2
 
4
3
  module I18n::Tasks::Scanners
@@ -7,6 +6,8 @@ module I18n::Tasks::Scanners
7
6
  # Caveat: scope is only detected when it is the first argument
8
7
  class PatternWithScopeScanner < PatternScanner
9
8
 
9
+ protected
10
+
10
11
  def default_pattern
11
12
  # capture the first argument and scope argument if present
12
13
  /#{super}
@@ -14,8 +15,6 @@ module I18n::Tasks::Scanners
14
15
  /x
15
16
  end
16
17
 
17
- protected
18
-
19
18
  # Given
20
19
  # @param [MatchData] match
21
20
  # @param [String] path
@@ -1,4 +1,3 @@
1
- # coding: utf-8
2
1
  module I18n
3
2
  module Tasks
4
3
  module Scanners
@@ -7,13 +6,13 @@ module I18n
7
6
  # @param path [String] path to the file containing the key
8
7
  # @return [String] absolute version of the key
9
8
  def absolutize_key(key, path, roots = relative_roots, closest_method = "")
9
+ fail 'roots argument is required' if roots.nil?
10
10
  normalized_path = File.expand_path(path)
11
11
  path_root(normalized_path, roots) or
12
- raise CommandError.new(
12
+ fail CommandError.new(
13
13
  "Error scanning #{normalized_path}: cannot resolve relative key
14
14
  \"#{key}\".\nSet search.relative_roots in config/i18n-tasks.yml
15
- (currently #{relative_roots.inspect})"
16
- )
15
+ (currently #{roots.inspect})")
17
16
 
18
17
  prefix_key_based_on_path(key, normalized_path, roots, closest_method: closest_method)
19
18
  end
@@ -0,0 +1,15 @@
1
+ require 'i18n/tasks/scanners/key_occurrences'
2
+
3
+ module I18n::Tasks::Scanners
4
+ # Describes the API of a scanner.
5
+ #
6
+ # @abstract
7
+ # @since 0.9.0
8
+ class Scanner
9
+ # @abstract
10
+ # @return [Array<KeyOccurrences>] the keys found by this scanner and their occurrences.
11
+ def keys
12
+ raise 'Unimplemented'
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,43 @@
1
+ require 'i18n/tasks/scanners/scanner'
2
+
3
+ module I18n::Tasks::Scanners
4
+ # Run multiple {Scanner Scanners} and merge their results.
5
+ # @note The scanners are run concurrently. A thread is spawned per each scanner.
6
+ # @since 0.9.0
7
+ class ScannerMultiplexer < Scanner
8
+ # @param scanners [Array<Scanner>]
9
+ def initialize(scanners:)
10
+ @scanners = scanners
11
+ end
12
+
13
+ # Collect the results of all the scanners. Occurrences of a key from multiple scanners are merged.
14
+ #
15
+ # @note The scanners are run concurrently. A thread is spawned per each scanner.
16
+ # @return (see Scanner#keys)
17
+ def keys
18
+ collect_results.inject({}) { |results_by_key, key_occurences|
19
+ key_occurences.each do |key_occurrence|
20
+ (results_by_key[key_occurrence.key] ||= []) << key_occurrence.occurrences
21
+ end
22
+ results_by_key
23
+ }.map { |key, all_occurrences|
24
+ occurrences = all_occurrences.flatten(1)
25
+ occurrences.sort_by!(&:path)
26
+ occurrences.uniq!
27
+ KeyOccurrences.new(key: key, occurrences: occurrences)
28
+ }
29
+ end
30
+
31
+ private
32
+
33
+ # @return Array<Array<KeyOccurrences>>
34
+ def collect_results
35
+ return [@scanners[0].keys] if @scanners.length == 1
36
+ Array.new(@scanners.length).tap do |results|
37
+ @scanners.map.with_index { |scanner, i|
38
+ Thread.start { results[i] = scanner.keys }
39
+ }.each(&:join)
40
+ end
41
+ end
42
+ end
43
+ end