rails_translation_manager 0.1.0 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9053fb88fb8e337bba059e6b16a13f16ffc058de0cc87b6be236c78d92a54dcd
-  data.tar.gz: 626c3ea8e720db73bce5ca3618f58d4ce121b7b845dd66e282572c4f9830406a
+  metadata.gz: 6e7ef3e878a1b5ffc7ae48ad8f683887e79073b3f34a5a3439656f5f7abb6e20
+  data.tar.gz: 2aac75f9cae7275812da73da0973de515baf0d78eb63e64182e683f8717696ca
 SHA512:
-  metadata.gz: 6d5334a90e4b52b7c4976846984dc83b9adb8ed2e3724eb181dde8f86118f72fc42375466285487f9eb3f1dafc78799662fe513cacdaad5902ededdbad54f2c1
-  data.tar.gz: ae8c750799c3882ce9b11408968e8322ea37f0bffcb3ca76ef8d117de8ab8ce30e65661e4a5bb1e297db106ff29436c03bc169bf31b6187f9dec309da4bcab7d
+  metadata.gz: f0b2fa6abaadb21ea912976b1092c6459ea0194a8e19092c522e38224da218395d230936d0d4a0aa830e3d37bd9f6eb6ac1e4909bd5406369c0aa99d1a4d13f4
+  data.tar.gz: 0dbbefe8208052b63f5ddc80734340ed7985b199f9113aaf5457c48d50e34e889e7379a20c61963b002343f4fb0d35aacc0307b442a15a2b7fbdea77c25349dd

data/.gitignore

@@ -6,7 +6,7 @@
 /doc/
 /pkg/
 /spec/reports/
-/tmp/
+/tmp/*
 *.bundle
 *.so
 *.o

data/.ruby-version

@@ -1 +1 @@
-2.5
+2.6.6

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 1.1.2
+
+Handle importing files that contain Byte Order Marks. https://github.com/alphagov/rails_translation_manager/pull/27
+
+## 1.1.1
+
+Fix Rails Translation Manager / Rails naming clash for class. https://github.com/alphagov/rails_translation_manager/pull/26
+
+## 1.1.0
+
+Allow multiple files per language to be imported. https://github.com/alphagov/rails_translation_manager/pull/20
+
+## 1.0.0
+
+Adds logic to verify locale files are in sync with each other and have the
+correct plural forms. Also introduces RSpec into the Gem which will replace
+Minitest in the coming iterations.
+
+Integrates RTM with I18n-tasks. Adds wrappers around `add-missing` & `normalize` tasks, adds Cleaner class to remove the whitespace added by i18n-tasks, adds tests and byebug gem as a debugging tool.
+
 ## 0.1.0
 
 Don't change the $LOAD_PATH in translation.rake.

data/Jenkinsfile

@@ -3,5 +3,7 @@
 library("govuk")
 
 node {
-  govuk.buildProject()
+  govuk.buildProject(
+    cleanWorkspace: true,
+  )
 }

data/Rakefile

@@ -1,11 +1,17 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
+# frozen_string_literal: true
 
-Rake::TestTask.new("test") do |t|
-  t.description = "Run tests"
-  t.libs << "test"
-  t.test_files = FileList["test/**/*_test.rb"]
+require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'rails_i18n'
+
+Rake::TestTask.new('test') do |t|
+  t.description = 'Run tests'
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
   t.verbose = true
 end
 
-task :default => :test
+RSpec::Core::RakeTask.new(:spec)
+
+task default: %i[spec test]

data/config/locales/plurals.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+{
+  # Dari - this isn't an iso code. Probably should be 'prs' as per ISO 639-3.
+  dr: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Latin America and Caribbean Spanish
+  "es-419": { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Scottish Gaelic
+  gd: { i18n: { plural: { keys: %i[one two few other],
+                          rule: lambda do |n|
+                            if [1, 11].include?(n)
+                              :one
+                            elsif [2, 12].include?(n)
+                              :two
+                            elsif [3, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 17, 18, 19].include?(n)
+                              :few
+                            else
+                              :other
+                            end
+                          end } } },
+  # Gujarati
+  gu: { i18n: { plural: { keys: %i[one other], rule: ->(n) { [0, 1].include?(n) ? :one : :other } } } },
+  # Armenian
+  hy: { i18n: { plural: { keys: %i[one other], rule: ->(n) { [0, 1].include?(n) ? :one : :other } } } },
+  # Kazakh
+  kk: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Pashto
+  ps: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Punjabi Shahmukhi
+  "pa-pk": { i18n: { plural: { keys: %i[one other], rule: ->(n) { [0, 1].include?(n) ? :one : :other } } } },
+  # Sinhalese
+  si: { i18n: { plural: { keys: %i[one other], rule: ->(n) { [0, 1].include?(n) ? :one : :other } } } },
+  # Somali
+  so: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Albanian
+  sq: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Norwegian
+  no: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Tamil
+  ta: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Turkmen
+  tk: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Uzbek
+  uz: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Chinese Hong Kong
+  'zh-hk' => { i18n: { plural: { keys: %i[other], rule: -> { :other } } } },
+  # Chinese Taiwan
+  'zh-tw' => { i18n: { plural: { keys: %i[other], rule: -> { :other } } } }
+}

data/lib/rails_translation_manager/cleaner.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module RailsTranslationManager
+  class Cleaner
+
+    def initialize(path)
+      @path = path
+    end
+
+    def clean
+      `find #{@path} -type f -exec perl -p -i -e \"s/[ \t]$//g\" {} \\;`
+    end
+  end
+end

data/lib/rails_translation_manager/i18n_tasks_option_parser.rb

@@ -0,0 +1,10 @@
+class RailsTranslationManager::I18nTasksOptionParser
+  def initialize(task_options, locale)
+    @task_options = task_options
+    @locale = locale
+  end
+
+  def with_optional_locale
+    @locale ? @task_options << ["-l", @locale] : @task_options
+  end
+end

data/lib/rails_translation_manager/importer.rb

@@ -5,36 +5,39 @@ require_relative "yaml_writer"
 class RailsTranslationManager::Importer
   include YAMLWriter
 
-  def initialize(locale, csv_path, import_directory)
-    @csv_path = csv_path
+  attr_reader :locale, :csv_path, :import_directory, :multiple_files_per_language
+
+  def initialize(locale:, csv_path:, import_directory:, multiple_files_per_language:)
     @locale = locale
+    @csv_path = csv_path
     @import_directory = import_directory
+    @multiple_files_per_language = multiple_files_per_language
   end
 
   def import
-    csv = CSV.read(@csv_path, headers: true, header_converters: :downcase)
-    data = {}
-    csv.each do |row|
+    csv = CSV.read(csv_path, encoding: "bom|utf-8", headers: true, header_converters: :downcase)
+
+    multiple_files_per_language ? import_csv_into_multiple_files(csv) : import_csv(csv)
+  end
+
+  private
+
+  def import_csv(csv, import_yml_path = File.join(import_directory, "#{locale}.yml"))
+    data = csv.each_with_object({}) do |row, hash|
       key = row["key"]
       key_parts = key.split(".")
       if key_parts.length > 1
-        leaf_node = (data[key_parts.first] ||= {})
+        leaf_node = (hash[key_parts.first] ||= {})
         key_parts[1..-2].each do |part|
           leaf_node = (leaf_node[part] ||= {})
         end
         leaf_node[key_parts.last] = parse_translation(row["translation"])
       else
-        data[key_parts.first] = parse_translation(row["translation"])
+        hash[key_parts.first] = parse_translation(row["translation"])
       end
     end
 
-    write_yaml(import_yml_path, {@locale.to_s => data})
-  end
-
-  private
-
-  def import_yml_path
-    File.join(@import_directory, "#{@locale}.yml")
+    write_yaml(import_yml_path, { locale.to_s => data })
   end
 
   def parse_translation(translation)
@@ -55,4 +58,19 @@ class RailsTranslationManager::Importer
       translation
     end
   end
+
+  def import_csv_into_multiple_files(csv)
+    group_csv_by_file(csv).each do |group|
+      language_dir =  File.join(import_directory, locale)
+
+      Dir.mkdir(language_dir) unless Dir.exists?(language_dir)
+
+      import_yml_path = File.join(import_directory, locale, "#{group[0]}.yml")
+      import_csv(group[1], import_yml_path)
+    end
+  end
+
+  def group_csv_by_file(csv)
+    csv.group_by { |row| row["key"].split(".").first }
+  end
 end

data/lib/rails_translation_manager/locale_checker/all_locales.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+class AllLocales
+  attr_reader :locale_path
+
+  def initialize(locale_path)
+    @locale_path = locale_path
+  end
+
+  def generate
+    paths = locale_file_paths.compact
+
+    raise NoLocaleFilesFound, "No locale files found for the supplied path" if paths.blank?
+
+    paths.flat_map do |locale_group|
+      {
+        locale: locale_group[:locale],
+        keys: all_keys_for_locale(locale_group)
+      }
+    end
+  end
+
+  private
+
+  def locale_file_paths
+    I18n.available_locales.map do |locale|
+      grouped_paths = Dir[locale_path].select do |path|
+        locale = locale.downcase
+
+        path =~ %r{/#{locale}/} || path =~ %r{/#{locale}\.yml$}
+      end
+
+      { locale: locale.downcase, paths: grouped_paths } if grouped_paths.present?
+    end
+  end
+
+  def all_keys_for_locale(locale_group)
+    locale_group[:paths].flat_map do |path|
+      keys_from_file(file_path: path)
+    end
+  end
+
+  def keys_from_file(locale_hash: nil, key_chain: nil, locale_keys: [], file_path: nil)
+    locale_hash ||= YAML.load_file(file_path)
+    keys = locale_hash.keys
+    keys.each do |key|
+      if locale_hash.fetch(key).is_a?(Hash)
+        keys_from_file(locale_hash: locale_hash.fetch(key), key_chain: "#{key_chain}.#{key}", locale_keys: locale_keys)
+      else
+        keys.each do |final_key|
+          locale_keys << "#{key_chain}.#{final_key}"
+        end
+      end
+    end
+
+    # remove locale prefix from keys, e.g: ".en.browse.page" -> "browse.page"
+    locale_keys.uniq.map { |key| key.split(".")[2..].join(".") }
+  end
+
+  class NoLocaleFilesFound < StandardError; end
+end

data/lib/rails_translation_manager/locale_checker/base_checker.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class BaseChecker
+  attr_reader :all_locales
+
+  def initialize(all_locales)
+    @all_locales = all_locales
+  end
+
+  def report
+    raise "You must define a `report` method in the child class!"
+  end
+end

data/lib/rails_translation_manager/locale_checker/incompatible_plurals.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+class IncompatiblePlurals < BaseChecker
+  include LocaleCheckerHelper
+
+  def report
+    format_plural_errors if plural_errors.present?
+  end
+
+  private
+
+  def format_plural_errors
+    <<~OUTPUT.chomp
+      \e[31m[ERROR]\e[0m Incompatible plural forms, for:
+
+      #{plural_errors.join("\n\n")}
+
+      \e[1mIf the keys reported above are not plurals, rename them avoiding plural keywords: #{PLURAL_KEYS}\e[22m
+    OUTPUT
+  end
+
+  def plural_errors
+    @plural_errors ||= grouped_plural_keys_for_locale.flat_map do |plurals|
+      plural_form = all_plural_forms[plurals[:locale]]
+
+      if plural_form.blank?
+        "- \e[31m[ERROR]\e[0m Please add plural form for '#{plurals[:locale]}' <link to future documentation>"
+      else
+        incompatible_plurals(plurals, plural_form)
+      end
+    end
+  end
+
+  def incompatible_plurals(plurals, plural_form)
+    error_messages = plurals[:groups].map do |plural_group|
+      actual_plural_form = plural_group[:keys].sort
+
+      next if actual_plural_form == plural_form.sort
+
+      "- '#{plurals[:locale]}', with parent '#{plural_group[:parent]}'. Expected: #{plural_form}, actual: #{actual_plural_form}"
+    end
+
+    error_messages.compact
+  end
+
+  def grouped_plural_keys_for_locale
+    all_locales.map do |locale|
+      {
+        locale: locale[:locale],
+        groups: grouped_plural_keys(only_plurals(locale[:keys]))
+      }
+    end
+  end
+
+  def grouped_plural_keys(keys)
+    # %w[parent.key parent.other_key] -> [{ parent: "parent", keys: %w[key other_key] }]
+    group_by_parent_keys(keys).map do |plural_key_group|
+      {
+        parent: plural_key_group.first,
+        keys: plural_key_group.last.map { |key_chain| key_chain.split(".").last.to_sym }
+      }
+    end
+  end
+
+  def group_by_parent_keys(keys)
+    # %w[parent.key parent.other_key] -> { "parent" => %w[parent.key parent.other_key] }
+    keys.group_by do |key|
+      key.split('.')[0..-2].join('.')
+    end
+  end
+
+  def only_plurals(keys)
+    keys.select { |key| key_is_plural?(key) }
+  end
+
+  def all_plural_forms
+    @all_plural_forms ||= PluralForms.all
+  end
+end

data/lib/rails_translation_manager/locale_checker/locale_checker_helper.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module LocaleCheckerHelper
+  PLURAL_KEYS = %w[zero one two few many other].freeze
+
+  def exclude_plurals(keys)
+    keys.reject { |key| key_is_plural?(key) }
+  end
+
+  def group_keys(locales)
+    locales.keys.group_by do |key|
+      locales[key]
+    end
+  end
+
+  def english_keys_excluding_plurals(all_locales)
+    exclude_plurals(all_locales.find { |locale| locale[:locale] == :en }[:keys])
+  end
+
+  def key_is_plural?(key)
+    PLURAL_KEYS.include?(key.split(".").last)
+  end
+end

data/lib/rails_translation_manager/locale_checker/missing_english_locales.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class MissingEnglishLocales < BaseChecker
+  include LocaleCheckerHelper
+
+  def report
+    grouped_missing_keys = group_keys(missing_english_locales)
+
+    format_missing_english_locales(grouped_missing_keys) if grouped_missing_keys.present?
+  end
+
+  private
+
+  def format_missing_english_locales(keys)
+    formatted_keys = keys.to_a.map do |group|
+      "\e[1mMissing English keys:\e[22m #{group[0]}\n\e[1mFound in:\e[22m #{group[1]}"
+    end
+
+    "\e[31m[ERROR]\e[0m Missing English locales, either remove these keys from the foreign locales or add them to the English locales\n\n#{formatted_keys.join("\n\n")}"
+  end
+
+  def missing_english_locales
+    all_locales.each_with_object({}) do |locale, hsh|
+      missing_keys = exclude_plurals(locale[:keys]) - english_keys_excluding_plurals(all_locales)
+
+      next if missing_keys.blank?
+
+      hsh[locale[:locale]] = missing_keys
+    end
+  end
+end

data/lib/rails_translation_manager/locale_checker/missing_foreign_locales.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class MissingForeignLocales < BaseChecker
+  include LocaleCheckerHelper
+
+  def report
+    grouped_missing_keys = group_keys(missing_foreign_locales)
+
+    format_missing_foreign_locales(grouped_missing_keys) if grouped_missing_keys.present?
+  end
+
+  private
+
+  def format_missing_foreign_locales(keys)
+    formatted_keys = keys.to_a.map do |group|
+      "\e[1mMissing foreign keys:\e[22m #{group[0]}\n\e[1mAbsent from:\e[22m #{group[1]}"
+    end
+
+    "\e[31m[ERROR]\e[0m Missing foreign locales, either add these keys to the foreign locales or remove them from the English locales\e[0m\n\n#{formatted_keys.join("\n\n")}"
+  end
+
+  def missing_foreign_locales
+    all_locales.each_with_object({}) do |locale, hsh|
+      missing_keys = english_keys_excluding_plurals(all_locales) - exclude_plurals(locale[:keys])
+
+      next if missing_keys.blank?
+
+      hsh[locale[:locale]] = missing_keys
+    end
+  end
+end

data/lib/rails_translation_manager/locale_checker/plural_forms.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class PluralForms
+  def self.all
+    I18n.available_locales.each_with_object({}) do |locale, hsh|
+      begin
+        # fetches plural form (rule) from rails-i18n for locale
+        plural_form = I18n.with_locale(locale) { I18n.t!("i18n.plural.keys") }.sort
+      rescue I18n::MissingTranslationData
+        plural_form = nil
+      end
+
+      hsh[locale.downcase] = plural_form
+    end
+  end
+end

data/lib/rails_translation_manager/locale_checker.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module RailsTranslationManager
+  class LocaleChecker
+    attr_reader :locale_path
+
+    CHECKER_CLASSES = [MissingEnglishLocales,
+                       MissingForeignLocales,
+                       IncompatiblePlurals].freeze
+
+    def initialize(locale_path)
+      @locale_path = locale_path
+    end
+
+    def validate_locales
+      output_result
+    rescue AllLocales::NoLocaleFilesFound => e
+      puts e
+      false
+    end
+
+    private
+
+    def output_result
+      errors = checker_errors.compact
+
+      if errors.blank?
+        puts "Locale files are in sync, nice job!"
+        true
+      else
+        errors.each do |error_message|
+          puts "\n"
+          puts error_message
+          puts "\n"
+        end
+        false
+      end
+    end
+
+    def checker_errors
+      CHECKER_CLASSES.flat_map do |checker|
+        checker.new(all_locales).report
+      end
+    end
+
+    def all_locales
+      @all_locales ||= AllLocales.new(locale_path).generate
+    end
+  end
+end

data/lib/rails_translation_manager/version.rb

@@ -1,3 +1,3 @@
 module RailsTranslationManager
-  VERSION = "0.1.0"
+  VERSION = "1.1.2"
 end

data/lib/rails_translation_manager.rb

@@ -1,9 +1,27 @@
+# frozen_string_literal: true
+
 require "rails_translation_manager/version"
 require "rails_translation_manager/railtie" if defined?(Rails)
 require "rails-i18n"
 
+require "rails_translation_manager/locale_checker/base_checker"
+require "rails_translation_manager/locale_checker/locale_checker_helper"
+require "rails_translation_manager/locale_checker/missing_foreign_locales"
+require "rails_translation_manager/locale_checker/missing_english_locales"
+require "rails_translation_manager/locale_checker/plural_forms"
+require "rails_translation_manager/locale_checker/incompatible_plurals"
+require "rails_translation_manager/locale_checker/all_locales"
+require "rails_translation_manager/locale_checker"
+require "rails_translation_manager/cleaner"
+require "rails_translation_manager/exporter"
+require "rails_translation_manager/importer"
+
 module RailsTranslationManager
-  autoload :Exporter, "rails_translation_manager/exporter"
-  autoload :Importer, "rails_translation_manager/importer"
-  autoload :Stealer, "rails_translation_manager/stealer"
+  rails_i18n_path = Gem::Specification.find_by_name("rails-i18n").gem_dir
+  rails_translation_manager = Gem::Specification.find_by_name("rails_translation_manager").gem_dir
+
+  I18n.load_path.concat(
+    Dir["#{rails_i18n_path}/rails/pluralization/*.rb"],
+    ["#{rails_translation_manager}/config/locales/plurals.rb"]
+  )
 end