rails_translation_manager 0.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: bcf654f074b115a7c80d173e821333d2b8b35b01
-  data.tar.gz: 84cea8e22ca03313c47e7432be605cee30905391
+SHA256:
+  metadata.gz: 0b6a096ee29746cead2529ea0cb83660c0e0217b710e085790969f203f246871
+  data.tar.gz: 3c13cb54190c17f1382246827c120cf928e7763a9de4b4724ec36ffe6005fc62
 SHA512:
-  metadata.gz: 6823237f243a4127c4c14ece436ec69212c1e481a1d11d8329649cfbcdc61177bf73f0097c1978fed353c863c83949f9eee62f616c16086f28757ce6645b5f0b
-  data.tar.gz: 2d8bc1c02272590a00a8bcbec3d002cec989f27c69e5025e65729ebf56c198089d0f29c8f7b87aaab5434642b28f2f11037a9596babdb1e096dd9ff8bda1a850
+  metadata.gz: 37abc2c32de6245dc3d1cd57da56f14ab3c68f88f9b3802a65735d3c36c02984f823d45ff177729daad6b2234929d03535ca28239a4188889a14be60f35875df
+  data.tar.gz: 73f08794d826203ba2be01695751d742bd7a546dee1bf9f8946c98cf57350a48651addd5eef53cfcd9b23303900d6efe0dcd41fb0ae81d67ba060ed851b5061a

data/.gitignore

@@ -6,7 +6,7 @@
 /doc/
 /pkg/
 /spec/reports/
-/tmp/
+/tmp/*
 *.bundle
 *.so
 *.o

data/.ruby-version

@@ -1 +1 @@
-2.1.4
+2.6.6

data/CHANGELOG.md

@@ -0,0 +1,23 @@
+## 1.1.0
+
+Allow multiple files per language to be imported. https://github.com/alphagov/rails_translation_manager/pull/20
+
+## 1.0.0
+
+Adds logic to verify locale files are in sync with each other and have the
+correct plural forms. Also introduces RSpec into the Gem which will replace
+Minitest in the coming iterations.
+
+Integrates RTM with I18n-tasks. Adds wrappers around `add-missing` & `normalize` tasks, adds Cleaner class to remove the whitespace added by i18n-tasks, adds tests and byebug gem as a debugging tool.
+
+## 0.1.0
+
+Don't change the $LOAD_PATH in translation.rake.
+
+## 0.0.2
+
+Added `steal` rake tasks to copy translations from another app.
+
+## 0.0.1
+
+Initial release.

data/Jenkinsfile

@@ -0,0 +1,9 @@
+#!/usr/bin/env groovy
+
+library("govuk")
+
+node {
+  govuk.buildProject(
+    cleanWorkspace: true,
+  )
+}

data/LICENSE.txt

@@ -1,5 +1,5 @@
 The MIT License (MIT)
-Copyright (C) 2015 HM Government (Government Digital Service)
+Copyright (C) 2015 Crown Copyright (Government Digital Service)
 
 MIT License
 

data/README.md

@@ -14,7 +14,7 @@ Support for translation workflow in rails applications.
                      into a translated string of text, for example `Hello %{name}`
                      would become `Hello Sarah` if the variable `name` had the
                      value `Sarah`.
-- **YAML**: yet another markup language, a textual data format used for storing
+- **YAML**: YAML Ain't Markup Language, a textual data format used (in this case) for storing
             translation strings in rails applications
 
 ## Technical documentation
@@ -124,6 +124,48 @@ $ rake translation:validate
 Success! No unexpected interpolation keys found.
 ```
 
+### Stealing translations from another app
+
+A third feature is the ability to "steal" one or more locales from an existing
+application. This functionality works by providing a mapping file, which defines
+how the translation keys in the the source app's files map to those in the app
+the gem is installed in.
+
+For example, given a locale file like this in the app to "steal" from:
+
+```yaml
+es:
+  document:
+    type:
+      case_study: Caso de estudio
+      consultation: Consulta
+```
+
+and a mapping file like this:
+
+```yaml
+document.type: content_item.format
+```
+
+running `rake translation:steal[es,../other_app,mapping_file_path.yml]` will
+result in the following locale file being created:
+
+```yaml
+es:
+  content_item:
+    format:
+      case_study: Caso de estudio
+      consultation: Consulta
+```
+
+The mapping file can live anywhere, as long as the full path (including filename)
+is given in the rake task invocation.
+
+The process will preserve data already in the output file if it is not
+referenced in the mapping, but will always override data belonging to keys
+that are in the mapping.
+
+
 ### Rake command reference
 
 #### Export a specific locale to CSV
@@ -150,6 +192,8 @@ rake translation:import[locale,path]
 rake translation:import:all[directory]
 ```
 
+#### 
+
 #### Regenerate all locales from the EN locale - run this after adding keys
 
 ```
@@ -162,6 +206,18 @@ rake translation:regenerate[directory]
 rake translation:validate
 ```
 
+#### Steal a specific locale file from another app
+
+```
+rake translation:steal[locale,source_app_path,mapping_file_path]
+```
+
+#### Steal all locale files from another app
+
+```
+rake translation:steal:all[source_app_path,mapping_file_path]
+```
+
 ### Running the test suite
 
 To run the test suite just run `bundle exec rake` from within the

data/Rakefile

@@ -1,12 +1,17 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
-
-Rake::TestTask.new("test") do |t|
-  t.description = "Run tests"
-  t.ruby_opts << "-rubygems"
-  t.libs << "test"
-  t.test_files = FileList["test/**/*_test.rb"]
+# frozen_string_literal: true
+
+require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'rails_i18n'
+
+Rake::TestTask.new('test') do |t|
+  t.description = 'Run tests'
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
   t.verbose = true
 end
 
-task :default => :test
+RSpec::Core::RakeTask.new(:spec)
+
+task default: %i[spec test]

data/config/locales/plurals.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+{
+  # Dari - this isn't an iso code. Probably should be 'prs' as per ISO 639-3.
+  dr: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Latin America and Caribbean Spanish
+  "es-419": { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Scottish Gaelic
+  gd: { i18n: { plural: { keys: %i[one two few other],
+                          rule: lambda do |n|
+                            if [1, 11].include?(n)
+                              :one
+                            elsif [2, 12].include?(n)
+                              :two
+                            elsif [3, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 17, 18, 19].include?(n)
+                              :few
+                            else
+                              :other
+                            end
+                          end } } },
+  # Gujarati
+  gu: { i18n: { plural: { keys: %i[one other], rule: ->(n) { [0, 1].include?(n) ? :one : :other } } } },
+  # Armenian
+  hy: { i18n: { plural: { keys: %i[one other], rule: ->(n) { [0, 1].include?(n) ? :one : :other } } } },
+  # Kazakh
+  kk: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Pashto
+  ps: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Punjabi Shahmukhi
+  "pa-pk": { i18n: { plural: { keys: %i[one other], rule: ->(n) { [0, 1].include?(n) ? :one : :other } } } },
+  # Sinhalese
+  si: { i18n: { plural: { keys: %i[one other], rule: ->(n) { [0, 1].include?(n) ? :one : :other } } } },
+  # Somali
+  so: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Albanian
+  sq: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Norwegian
+  no: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Tamil
+  ta: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Turkmen
+  tk: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Uzbek
+  uz: { i18n: { plural: { keys: %i[one other], rule: ->(n) { n == 1 ? :one : :other } } } },
+  # Chinese Hong Kong
+  'zh-hk' => { i18n: { plural: { keys: %i[other], rule: -> { :other } } } },
+  # Chinese Taiwan
+  'zh-tw' => { i18n: { plural: { keys: %i[other], rule: -> { :other } } } }
+}

data/lib/rails_translation_manager/cleaner.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module RailsTranslationManager
+  class Cleaner
+
+    def initialize(path)
+      @path = path
+    end
+
+    def clean
+      `find #{@path} -type f -exec perl -p -i -e \"s/[ \t]$//g\" {} \\;`
+    end
+  end
+end

data/lib/rails_translation_manager/importer.rb

@@ -1,42 +1,43 @@
 require "yaml"
 require "csv"
+require_relative "yaml_writer"
 
 class RailsTranslationManager::Importer
-  def initialize(locale, csv_path, import_directory)
-    @csv_path = csv_path
+  include YAMLWriter
+
+  attr_reader :locale, :csv_path, :import_directory, :multiple_files_per_language
+
+  def initialize(locale:, csv_path:, import_directory:, multiple_files_per_language:)
     @locale = locale
+    @csv_path = csv_path
     @import_directory = import_directory
+    @multiple_files_per_language = multiple_files_per_language
   end
 
   def import
-    csv = CSV.read(@csv_path, headers: true, header_converters: :downcase)
-    data = {}
-    csv.each do |row|
+    csv = CSV.read(csv_path, headers: true, header_converters: :downcase)
+
+    multiple_files_per_language ? import_csv_into_multiple_files(csv) : import_csv(csv)
+  end
+
+  private
+
+  def import_csv(csv, import_yml_path = File.join(import_directory, "#{locale}.yml"))
+    data = csv.each_with_object({}) do |row, hash|
       key = row["key"]
       key_parts = key.split(".")
       if key_parts.length > 1
-        leaf_node = (data[key_parts.first] ||= {})
+        leaf_node = (hash[key_parts.first] ||= {})
         key_parts[1..-2].each do |part|
           leaf_node = (leaf_node[part] ||= {})
         end
         leaf_node[key_parts.last] = parse_translation(row["translation"])
       else
-        data[key_parts.first] = parse_translation(row["translation"])
+        hash[key_parts.first] = parse_translation(row["translation"])
       end
     end
 
-    File.open(import_yml_path, "w") do |f|
-      yaml = {@locale.to_s => data}.to_yaml(separator: "")
-      yaml_without_header = yaml.split("\n").map { |l| l.gsub(/\s+$/, '') }[1..-1].join("\n")
-      f.write(yaml_without_header)
-      f.puts
-    end
-  end
-
-  private
-
-  def import_yml_path
-    File.join(@import_directory, "#{@locale}.yml")
+    write_yaml(import_yml_path, { locale.to_s => data })
   end
 
   def parse_translation(translation)
@@ -57,4 +58,19 @@ class RailsTranslationManager::Importer
       translation
     end
   end
+
+  def import_csv_into_multiple_files(csv)
+    group_csv_by_file(csv).each do |group|
+      language_dir =  File.join(import_directory, locale)
+
+      Dir.mkdir(language_dir) unless Dir.exists?(language_dir)
+
+      import_yml_path = File.join(import_directory, locale, "#{group[0]}.yml")
+      import_csv(group[1], import_yml_path)
+    end
+  end
+
+  def group_csv_by_file(csv)
+    csv.group_by { |row| row["key"].split(".").first }
+  end
 end

data/lib/rails_translation_manager/locale_checker/all_locales.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+class AllLocales
+  attr_reader :locale_path
+
+  def initialize(locale_path)
+    @locale_path = locale_path
+  end
+
+  def generate
+    paths = locale_file_paths.compact
+
+    raise NoLocaleFilesFound, "No locale files found for the supplied path" if paths.blank?
+
+    paths.flat_map do |locale_group|
+      {
+        locale: locale_group[:locale],
+        keys: all_keys_for_locale(locale_group)
+      }
+    end
+  end
+
+  private
+
+  def locale_file_paths
+    I18n.available_locales.map do |locale|
+      grouped_paths = Dir[locale_path].select do |path|
+        locale = locale.downcase
+
+        path =~ %r{/#{locale}/} || path =~ %r{/#{locale}\.yml$}
+      end
+
+      { locale: locale.downcase, paths: grouped_paths } if grouped_paths.present?
+    end
+  end
+
+  def all_keys_for_locale(locale_group)
+    locale_group[:paths].flat_map do |path|
+      keys_from_file(file_path: path)
+    end
+  end
+
+  def keys_from_file(locale_hash: nil, key_chain: nil, locale_keys: [], file_path: nil)
+    locale_hash ||= YAML.load_file(file_path)
+    keys = locale_hash.keys
+    keys.each do |key|
+      if locale_hash.fetch(key).is_a?(Hash)
+        keys_from_file(locale_hash: locale_hash.fetch(key), key_chain: "#{key_chain}.#{key}", locale_keys: locale_keys)
+      else
+        keys.each do |final_key|
+          locale_keys << "#{key_chain}.#{final_key}"
+        end
+      end
+    end
+
+    # remove locale prefix from keys, e.g: ".en.browse.page" -> "browse.page"
+    locale_keys.uniq.map { |key| key.split(".")[2..].join(".") }
+  end
+
+  class NoLocaleFilesFound < StandardError; end
+end

data/lib/rails_translation_manager/locale_checker/base_checker.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class BaseChecker
+  attr_reader :all_locales
+
+  def initialize(all_locales)
+    @all_locales = all_locales
+  end
+
+  def report
+    raise "You must define a `report` method in the child class!"
+  end
+end

data/lib/rails_translation_manager/locale_checker/incompatible_plurals.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+class IncompatiblePlurals < BaseChecker
+  include LocaleCheckerHelper
+
+  def report
+    format_plural_errors if plural_errors.present?
+  end
+
+  private
+
+  def format_plural_errors
+    <<~OUTPUT.chomp
+      \e[31m[ERROR]\e[0m Incompatible plural forms, for:
+
+      #{plural_errors.join("\n\n")}
+
+      \e[1mIf the keys reported above are not plurals, rename them avoiding plural keywords: #{PLURAL_KEYS}\e[22m
+    OUTPUT
+  end
+
+  def plural_errors
+    @plural_errors ||= grouped_plural_keys_for_locale.flat_map do |plurals|
+      plural_form = all_plural_forms[plurals[:locale]]
+
+      if plural_form.blank?
+        "- \e[31m[ERROR]\e[0m Please add plural form for '#{plurals[:locale]}' <link to future documentation>"
+      else
+        incompatible_plurals(plurals, plural_form)
+      end
+    end
+  end
+
+  def incompatible_plurals(plurals, plural_form)
+    error_messages = plurals[:groups].map do |plural_group|
+      actual_plural_form = plural_group[:keys].sort
+
+      next if actual_plural_form == plural_form.sort
+
+      "- '#{plurals[:locale]}', with parent '#{plural_group[:parent]}'. Expected: #{plural_form}, actual: #{actual_plural_form}"
+    end
+
+    error_messages.compact
+  end
+
+  def grouped_plural_keys_for_locale
+    all_locales.map do |locale|
+      {
+        locale: locale[:locale],
+        groups: grouped_plural_keys(only_plurals(locale[:keys]))
+      }
+    end
+  end
+
+  def grouped_plural_keys(keys)
+    # %w[parent.key parent.other_key] -> [{ parent: "parent", keys: %w[key other_key] }]
+    group_by_parent_keys(keys).map do |plural_key_group|
+      {
+        parent: plural_key_group.first,
+        keys: plural_key_group.last.map { |key_chain| key_chain.split(".").last.to_sym }
+      }
+    end
+  end
+
+  def group_by_parent_keys(keys)
+    # %w[parent.key parent.other_key] -> { "parent" => %w[parent.key parent.other_key] }
+    keys.group_by do |key|
+      key.split('.')[0..-2].join('.')
+    end
+  end
+
+  def only_plurals(keys)
+    keys.select { |key| key_is_plural?(key) }
+  end
+
+  def all_plural_forms
+    @all_plural_forms ||= PluralForms.all
+  end
+end

data/lib/rails_translation_manager/locale_checker/locale_checker_helper.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module LocaleCheckerHelper
+  PLURAL_KEYS = %w[zero one two few many other].freeze
+
+  def exclude_plurals(keys)
+    keys.reject { |key| key_is_plural?(key) }
+  end
+
+  def group_keys(locales)
+    locales.keys.group_by do |key|
+      locales[key]
+    end
+  end
+
+  def english_keys_excluding_plurals(all_locales)
+    exclude_plurals(all_locales.find { |locale| locale[:locale] == :en }[:keys])
+  end
+
+  def key_is_plural?(key)
+    PLURAL_KEYS.include?(key.split(".").last)
+  end
+end

data/lib/rails_translation_manager/locale_checker/missing_english_locales.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class MissingEnglishLocales < BaseChecker
+  include LocaleCheckerHelper
+
+  def report
+    grouped_missing_keys = group_keys(missing_english_locales)
+
+    format_missing_english_locales(grouped_missing_keys) if grouped_missing_keys.present?
+  end
+
+  private
+
+  def format_missing_english_locales(keys)
+    formatted_keys = keys.to_a.map do |group|
+      "\e[1mMissing English keys:\e[22m #{group[0]}\n\e[1mFound in:\e[22m #{group[1]}"
+    end
+
+    "\e[31m[ERROR]\e[0m Missing English locales, either remove these keys from the foreign locales or add them to the English locales\n\n#{formatted_keys.join("\n\n")}"
+  end
+
+  def missing_english_locales
+    all_locales.each_with_object({}) do |locale, hsh|
+      missing_keys = exclude_plurals(locale[:keys]) - english_keys_excluding_plurals(all_locales)
+
+      next if missing_keys.blank?
+
+      hsh[locale[:locale]] = missing_keys
+    end
+  end
+end

data/lib/rails_translation_manager/locale_checker/missing_foreign_locales.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class MissingForeignLocales < BaseChecker
+  include LocaleCheckerHelper
+
+  def report
+    grouped_missing_keys = group_keys(missing_foreign_locales)
+
+    format_missing_foreign_locales(grouped_missing_keys) if grouped_missing_keys.present?
+  end
+
+  private
+
+  def format_missing_foreign_locales(keys)
+    formatted_keys = keys.to_a.map do |group|
+      "\e[1mMissing foreign keys:\e[22m #{group[0]}\n\e[1mAbsent from:\e[22m #{group[1]}"
+    end
+
+    "\e[31m[ERROR]\e[0m Missing foreign locales, either add these keys to the foreign locales or remove them from the English locales\e[0m\n\n#{formatted_keys.join("\n\n")}"
+  end
+
+  def missing_foreign_locales
+    all_locales.each_with_object({}) do |locale, hsh|
+      missing_keys = english_keys_excluding_plurals(all_locales) - exclude_plurals(locale[:keys])
+
+      next if missing_keys.blank?
+
+      hsh[locale[:locale]] = missing_keys
+    end
+  end
+end

data/lib/rails_translation_manager/locale_checker/plural_forms.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class PluralForms
+  def self.all
+    I18n.available_locales.each_with_object({}) do |locale, hsh|
+      begin
+        # fetches plural form (rule) from rails-i18n for locale
+        plural_form = I18n.with_locale(locale) { I18n.t!("i18n.plural.keys") }.sort
+      rescue I18n::MissingTranslationData
+        plural_form = nil
+      end
+
+      hsh[locale.downcase] = plural_form
+    end
+  end
+end

data/lib/rails_translation_manager/locale_checker.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module RailsTranslationManager
+  class LocaleChecker
+    attr_reader :locale_path
+
+    CHECKER_CLASSES = [MissingEnglishLocales,
+                       MissingForeignLocales,
+                       IncompatiblePlurals].freeze
+
+    def initialize(locale_path)
+      @locale_path = locale_path
+    end
+
+    def validate_locales
+      output_result
+    rescue AllLocales::NoLocaleFilesFound => e
+      puts e
+      false
+    end
+
+    private
+
+    def output_result
+      errors = checker_errors.compact
+
+      if errors.blank?
+        puts "Locale files are in sync, nice job!"
+        true
+      else
+        errors.each do |error_message|
+          puts "\n"
+          puts error_message
+          puts "\n"
+        end
+        false
+      end
+    end
+
+    def checker_errors
+      CHECKER_CLASSES.flat_map do |checker|
+        checker.new(all_locales).report
+      end
+    end
+
+    def all_locales
+      @all_locales ||= AllLocales.new(locale_path).generate
+    end
+  end
+end

data/lib/rails_translation_manager/version.rb

@@ -1,3 +1,3 @@
 module RailsTranslationManager
-  VERSION = "0.0.1"
+  VERSION = "1.1.0"
 end

data/lib/rails_translation_manager/yaml_writer.rb

@@ -0,0 +1,17 @@
+require 'yaml'
+
+module YAMLWriter
+
+  # `to_yaml` outputs an initial '---\n', which is supposed to be a document
+  # separator. We don't want this in the written locale files, so we serialize
+  # to a string, remove the header and any trailing whitehspace, and write to
+  # the file.
+  def write_yaml(filepath, data)
+    File.open(filepath, "w") do |f|
+      yaml = data.to_yaml(separator: "")
+      yaml_without_header = yaml.split("\n").map { |l| l.gsub(/\s+$/, '') }[1..-1].join("\n")
+      f.write(yaml_without_header)
+      f.puts
+    end
+  end
+end