rails_translation_manager 1.0.0 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2407e2bb0a7f4cf4628d9a9f35bcfb668b085c23bb92418f708c09f571c1b92
-  data.tar.gz: e0584a3f42216bdc0b9589ab6bb26e7246d6856e0b33f7599ac1e6eed0070af8
+  metadata.gz: fbbde3519b97f6c9881189908af474ef24b5ad5dae1a3c6e3a590df8ca4b9885
+  data.tar.gz: f6ad0e6bf5584f6fa9db3b39d3a7d52c1b04786368c9297090a480690e2cf1e7
 SHA512:
-  metadata.gz: '0249a48778dade714f10f47d4185136da8a8a9d11313273d659817dba1968c01ed5756c42b87e0b1963d3f9a41c1316392317f7214137dca5af8e4962f311f4a'
-  data.tar.gz: 63553a493e8c36c8daa8fe0bdf388384c2bf80fca3e41fd038f4ed8f92078c4accfe18c9a9561c2dc9faa82a35e9cd4642c7b1894a21a1fbd2fb0302c18092a1
+  metadata.gz: befae5c31f13ce323a367033180d58d7969220cde8eded8dd213a0de5405fd82246f0b775c3761d45e2412b25699630538896d6b2ec9692a4f030ea751f9282c
+  data.tar.gz: 25979414b9918d47b58e3f87caa2ba967dcca0bc233c37854f270b8e93f0f95940b3c0f54b7f658ea4ed144b463b740569bebe4c0dfd98907a8e82d02c10fd6f

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 1.1.3
+
+Handle importing files that contain rows with a blank "key". https://github.com/alphagov/rails_translation_manager/pull/28
+
+## 1.1.2
+
+Handle importing files that contain Byte Order Marks. https://github.com/alphagov/rails_translation_manager/pull/27
+
+## 1.1.1
+
+Fix Rails Translation Manager / Rails naming clash for class. https://github.com/alphagov/rails_translation_manager/pull/26
+
+## 1.1.0
+
+Allow multiple files per language to be imported. https://github.com/alphagov/rails_translation_manager/pull/20
+
 ## 1.0.0
 
 Adds logic to verify locale files are in sync with each other and have the

data/Jenkinsfile

@@ -3,5 +3,7 @@
 library("govuk")
 
 node {
-  govuk.buildProject()
+  govuk.buildProject(
+    cleanWorkspace: true,
+  )
 }

data/lib/{tasks/translation_helper.rb → rails_translation_manager/i18n_tasks_option_parser.rb}

@@ -1,5 +1,4 @@
-class TranslationHelper
-
+class RailsTranslationManager::I18nTasksOptionParser
   def initialize(task_options, locale)
     @task_options = task_options
     @locale = locale

data/lib/rails_translation_manager/importer.rb

@@ -5,36 +5,49 @@ require_relative "yaml_writer"
 class RailsTranslationManager::Importer
   include YAMLWriter
 
-  def initialize(locale, csv_path, import_directory)
-    @csv_path = csv_path
+  attr_reader :locale, :csv_path, :import_directory, :multiple_files_per_language
+
+  def initialize(locale:, csv_path:, import_directory:, multiple_files_per_language:)
     @locale = locale
+    @csv_path = csv_path
     @import_directory = import_directory
+    @multiple_files_per_language = multiple_files_per_language
   end
 
   def import
-    csv = CSV.read(@csv_path, headers: true, header_converters: :downcase)
-    data = {}
-    csv.each do |row|
+    csv = reject_nil_keys(
+      CSV.read(csv_path, encoding: "bom|utf-8", headers: true, header_converters: :downcase)
+    )
+
+    multiple_files_per_language ? import_csv_into_multiple_files(csv) : import_csv(csv)
+  end
+
+  private
+
+  def import_csv(csv, import_yml_path = File.join(import_directory, "#{locale}.yml"))
+    data = csv.each_with_object({}) do |row, hash|
       key = row["key"]
       key_parts = key.split(".")
       if key_parts.length > 1
-        leaf_node = (data[key_parts.first] ||= {})
+        leaf_node = (hash[key_parts.first] ||= {})
         key_parts[1..-2].each do |part|
           leaf_node = (leaf_node[part] ||= {})
         end
         leaf_node[key_parts.last] = parse_translation(row["translation"])
       else
-        data[key_parts.first] = parse_translation(row["translation"])
+        hash[key_parts.first] = parse_translation(row["translation"])
       end
     end
 
-    write_yaml(import_yml_path, {@locale.to_s => data})
+    write_yaml(import_yml_path, { locale.to_s => data })
   end
 
-  private
-
-  def import_yml_path
-    File.join(@import_directory, "#{@locale}.yml")
+  def reject_nil_keys(csv)
+    csv.reject do |row|
+      nil_key = row["key"].nil?
+      puts "Invalid row: #{row.inspect} for csv_path: #{csv_path}\n" if nil_key == true
+      nil_key
+    end
   end
 
   def parse_translation(translation)
@@ -55,4 +68,19 @@ class RailsTranslationManager::Importer
       translation
     end
   end
+
+  def import_csv_into_multiple_files(csv)
+    group_csv_by_file(csv).each do |group|
+      language_dir =  File.join(import_directory, locale)
+
+      Dir.mkdir(language_dir) unless Dir.exists?(language_dir)
+
+      import_yml_path = File.join(import_directory, locale, "#{group[0]}.yml")
+      import_csv(group[1], import_yml_path)
+    end
+  end
+
+  def group_csv_by_file(csv)
+    csv.group_by { |row| row["key"].split(".").first }
+  end
 end

data/lib/rails_translation_manager/version.rb

@@ -1,3 +1,3 @@
 module RailsTranslationManager
-  VERSION = "1.0.0"
+  VERSION = "1.1.3"
 end

data/lib/rails_translation_manager.rb

@@ -13,12 +13,10 @@ require "rails_translation_manager/locale_checker/incompatible_plurals"
 require "rails_translation_manager/locale_checker/all_locales"
 require "rails_translation_manager/locale_checker"
 require "rails_translation_manager/cleaner"
+require "rails_translation_manager/exporter"
+require "rails_translation_manager/importer"
 
 module RailsTranslationManager
-  autoload :Exporter, "rails_translation_manager/exporter"
-  autoload :Importer, "rails_translation_manager/importer"
-  autoload :Stealer, "rails_translation_manager/stealer"
-
   rails_i18n_path = Gem::Specification.find_by_name("rails-i18n").gem_dir
   rails_translation_manager = Gem::Specification.find_by_name("rails_translation_manager").gem_dir
 

data/lib/tasks/translation.rake

@@ -1,17 +1,9 @@
-require "rails_translation_manager"
 require "i18n/tasks/cli"
-require_relative "../tasks/translation_helper"
+require_relative "../rails_translation_manager"
+require_relative "../rails_translation_manager/i18n_tasks_option_parser"
 
 namespace :translation do
 
-  desc "Regenerate all locales from the EN locale - run this after adding keys"
-  task(:regenerate, [:directory] => [:environment]) do |t, args|
-    directory = args[:directory] || "tmp/locale_csv"
-
-    Rake::Task["translation:export:all"].invoke(directory)
-    Rake::Task["translation:import:all"].invoke(directory)
-  end
-
   desc "Export a specific locale to CSV."
   task :export, [:directory, :base_locale, :target_locale] => [:environment] do |t, args|
     FileUtils.mkdir_p(args[:directory]) unless File.exist?(args[:directory])
@@ -38,62 +30,58 @@ namespace :translation do
   end
 
   desc "Import a specific locale CSV to YAML within the app."
-  task :import, [:locale, :path] => [:environment] do |t, args|
-    importer = RailsTranslationManager::Importer.new(args[:locale], args[:path], Rails.root.join("config", "locales"))
+  task :import, [:csv_path, :multiple_files_per_language] => [:environment] do |t, args|
+    import_dir = Rails.root.join("config", "locales")
+    csv_path = args[:csv_path]
+
+    importer = RailsTranslationManager::Importer.new(
+      locale: File.basename(args[:csv_path], ".csv"),
+      csv_path: csv_path,
+      import_directory: Rails.root.join("config", "locales"),
+      multiple_files_per_language: args[:multiple_files_per_language] || false
+    )
     importer.import
+
+    puts "\nImported CSV from: #{csv_path} to #{import_dir}"
   end
 
   namespace :import do
     desc "Import all locale CSV files to YAML within the app."
-    task :all, [:directory] => [:environment] do |t, args|
-      directory = args[:directory] || "tmp/locale_csv"
-      Dir[File.join(directory, "*.csv")].each do |csv_path|
-        locale = File.basename(csv_path, ".csv")
-        importer = RailsTranslationManager::Importer.new(locale, csv_path, Rails.root.join("config", "locales"))
+    task :all, [:csv_directory, :multiple_files_per_language] => [:environment] do |t, args|
+      directory = args[:csv_directory] || "tmp/locale_csv"
+      import_dir = Rails.root.join("config", "locales")
+
+      Dir[Rails.root.join(directory, "*.csv")].each do |csv_path|
+        importer = RailsTranslationManager::Importer.new(
+          locale: File.basename(csv_path, ".csv"),
+          csv_path: csv_path,
+          import_directory: import_dir,
+          multiple_files_per_language: args[:multiple_files_per_language] || false
+        )
         importer.import
       end
-    end
-  end
 
-  desc "Check translation files for errors"
-  task :validate do
-    require 'rails_translation_manager/validator'
-    logger = Logger.new(STDOUT)
-    validator = RailsTranslationManager::Validator.new(Rails.root.join('config', 'locales'), logger)
-    errors = validator.check!
-    if errors.any?
-      puts "Found #{errors.size} errors:"
-      puts errors.map(&:to_s).join("\n")
-    else
-      puts "Success! No unexpected interpolation keys found."
-    end
-  end
-
-  desc "Import and convert a locale file from another app."
-  task :steal, [:locale, :source_app_path, :mapping_file_path] do |t, args|
-    stealer = RailsTranslationManager::Stealer.new(args[:locale], args[:source_app_path], args[:mapping_file_path], Rails.root.join('config', 'locales'))
-    stealer.steal_locale
-  end
-
-  namespace :steal do
-    desc "Import and convert all locale files from another app."
-    task :all, [:source_app_path, :mapping_file_path] => [:environment] do |t, args|
-      I18n.available_locales.reject { |l| l == :en }.each do |locale|
-        stealer = RailsTranslationManager::Stealer.new(locale.to_s, args[:source_app_path], args[:mapping_file_path], Rails.root.join('config', 'locales'))
-        stealer.steal_locale
-      end
+      puts "\nImported all CSVs from: #{directory} to #{import_dir}"
     end
   end
 
   desc "Add missing translations"
   task(:add_missing, [:locale] => [:environment]) do |t, args|
-    I18n::Tasks::CLI.start(TranslationHelper.new(["add-missing", "--nil-value"], args[:locale]).with_optional_locale)
+    option_parser = RailsTranslationManager::I18nTasksOptionParser.new(
+      ["add-missing", "--nil-value"], args[:locale]
+    ).with_optional_locale
+
+    I18n::Tasks::CLI.start(option_parser)
     RailsTranslationManager::Cleaner.new(Rails.root.join("config", "locales")).clean
   end
 
   desc "Normalize translations"
   task(:normalize, [:locale] => [:environment]) do |t, args|
-    I18n::Tasks::CLI.start(TranslationHelper.new(["normalize"], args[:locale]).with_optional_locale)
+    option_parser = RailsTranslationManager::I18nTasksOptionParser.new(
+      ["normalize"], args[:locale]
+    ).with_optional_locale
+
+    I18n::Tasks::CLI.start(option_parser)
     RailsTranslationManager::Cleaner.new(Rails.root.join("config", "locales")).clean
   end
 end

data/rails_translation_manager.gemspec

@@ -18,9 +18,10 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "rails-i18n"
   spec.add_dependency "activesupport"
+  spec.add_dependency "csv", "~> 3.2"
   spec.add_dependency "i18n-tasks"
+  spec.add_dependency "rails-i18n"
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake", "~> 10.0"

data/spec/locales/importer/fr.csv

@@ -0,0 +1,9 @@
+key,source,translation
+world_location.type.country,Country,Pays
+world_location.fruit,"[Apples, Bananas, Pears]","[Pommes, Bananes, Poires]"
+world_location.things,nil,nil
+world_location.sentiment,:whatever,:bof
+shared.price,123,123
+shared.key1,is true,true
+shared.key2,is false,false
+,,

data/spec/locales/importer/hy_with_byte_order_mark.csv

@@ -0,0 +1,4 @@
+key,source,translation
+unpublishing.title,The page you're looking for is no longer available,Ձեր փնտրած էջն այլևս հասանելի չէ
+working_group.contact_details,Contact details,Կոնտակտային տվյալներ
+working_group.policies,Policies,Քաղաքականություններ

data/spec/rails_translation_manager/importer_spec.rb

@@ -0,0 +1,106 @@
+require "spec_helper"
+require "tmpdir"
+
+RSpec.describe RailsTranslationManager::Importer do
+  let(:import_directory) { Dir.mktmpdir }
+
+  it "imports CSV containing a byte order mark" do
+    importer = described_class.new(
+      locale: "hy",
+      csv_path: "spec/locales/importer/hy_with_byte_order_mark.csv",
+      import_directory: import_directory,
+      multiple_files_per_language: false
+    )
+    importer.import
+
+    expect(File).to exist(import_directory + "/hy.yml")
+  end
+
+  it "doesn't try to import a row with a blank key" do
+    importer = described_class.new(
+      locale: "hy",
+      csv_path: "spec/locales/importer/fr.csv",
+      import_directory: import_directory,
+      multiple_files_per_language: false
+    )
+
+    expect { importer.import }.to output(
+      "Invalid row: #<CSV::Row \"key\":nil \"source\":nil \"translation\":nil> for csv_path: spec/locales/importer/fr.csv\n"
+    ).to_stdout
+  end
+
+  context "when there is one locale file per language" do
+    let(:yaml_translation_data) { YAML.load_file(import_directory + "/fr.yml")["fr"] }
+
+    before do
+      importer = described_class.new(
+        locale: "fr",
+        csv_path: "spec/locales/importer/fr.csv",
+        import_directory: import_directory,
+        multiple_files_per_language: false
+      )
+      importer.import
+    end
+
+    it "creates one YAML file per language" do
+      expect(File).to exist(import_directory + "/fr.yml")
+    end
+
+    it "imports nested locales" do
+      expected = { "type" => { "country" => "Pays" } }
+      expect(yaml_translation_data).to include("world_location" => hash_including(expected))
+    end
+
+    it "imports arrays from CSV as arrays" do
+      expected =  { "fruit" => ["Pommes", "Bananes", "Poires"] }
+      expect(yaml_translation_data).to include("world_location" => hash_including(expected))
+    end
+
+    it "imports string 'nil' as nil" do
+      expected = { "things" => nil }
+      expect(yaml_translation_data).to include("world_location" => hash_including(expected))
+    end
+
+    it "imports string ':thing' as symbol" do
+      expected = { "sentiment" => :bof }
+      expect(yaml_translation_data).to include("world_location" => hash_including(expected))
+    end
+
+    it "imports integer strings as integers" do
+      expected = { "price" => 123 }
+      expect(yaml_translation_data).to include("shared" => hash_including(expected))
+    end
+
+    it "imports boolean values as booleans, not strings" do
+      expected = { "key1" => true, "key2" => false }
+      expect(yaml_translation_data).to include("shared" => hash_including(expected))
+    end
+  end
+
+  context "when there are multiple files per locale" do
+    before do
+      importer = described_class.new(
+        locale: "fr",
+        csv_path: "spec/locales/importer/fr.csv",
+        import_directory: import_directory,
+        multiple_files_per_language: true
+      )
+      importer.import
+    end
+
+    it "creates multiple YAML files per language in the language's directory" do
+      expect(File).to exist(import_directory + "/fr/world_location.yml")
+                  .and exist(import_directory + "/fr/shared.yml")
+    end
+
+    it "imports only 'world_location' locales to the relevant file" do
+      yaml_translation_data = YAML.load_file(import_directory + "/fr/world_location.yml")["fr"]
+      expect(yaml_translation_data).to match("world_location" => anything)
+    end
+
+    it "imports only 'shared' locales to the relevant file" do
+      yaml_translation_data = YAML.load_file(import_directory + "/fr/shared.yml")["fr"]
+      expect(yaml_translation_data).to match("shared" => anything)
+    end
+  end
+end

data/spec/tasks/translation_spec.rb

@@ -1,38 +1,127 @@
-require 'spec_helper'
-require_relative '../../spec/support/tasks'  
+require "spec_helper"
+require_relative "../../spec/support/tasks"
 
-describe 'rake tasks' do
+describe "rake tasks" do
   before do
     fake_rails = double()
-    fake_rails.stub(:root) { Pathname.new('spec') }
+    fake_rails.stub(:root) { Pathname.new("spec") }
     stub_const("Rails", fake_rails)
   end
 
-  describe 'translation:add_missing', type: :task do
-    let(:task) { Rake::Task["translation:add_missing"] }
-  
-    it 'is executed' do
-      expect { task.execute }.to output.to_stdout
+  describe "translation:import", type: :task do
+    let(:task) { Rake::Task["translation:import"] }
+    let(:csv_path) { "path/to/import/fr.csv" }
+    let!(:importer_instance) { stub_importer }
+
+    it "outputs to stdout" do
+      expect { task.execute(csv_path: csv_path) }
+        .to output("\nImported CSV from: #{csv_path} to #{Rails.root.join("config", "locales")}\n")
+        .to_stdout
+    end
+
+    it "calls the Importer class with the csv and import paths" do
+      task.execute(csv_path: csv_path)
+
+      expect(RailsTranslationManager::Importer)
+        .to have_received(:new)
+        .with(locale: "fr",
+              csv_path: csv_path,
+              import_directory: Rails.root.join("config", "locales"),
+              multiple_files_per_language: false)
+      expect(importer_instance).to have_received(:import)
+    end
+  end
+
+  describe "translation:import:all", type: :task do
+    let(:task) { Rake::Task["translation:import:all"] }
+    let(:csv_directory) { "locales/importer" }
+    let!(:importer_instance) { stub_importer }
+
+    it "outputs to stdout" do
+      expect { task.execute(csv_directory: csv_directory) }
+        .to output("\nImported all CSVs from: #{csv_directory} to #{Rails.root.join("config", "locales")}\n")
+        .to_stdout
+    end
+
+    it "calls the importer class for each target path" do
+      task.execute(csv_directory: csv_directory, multiple_files_per_language: true)
+      import_paths = Dir["spec/locales/importer/*.csv"]
+
+      import_paths.each do |csv_path|
+        expect(RailsTranslationManager::Importer)
+          .to have_received(:new)
+          .with(locale: File.basename(csv_path, ".csv"),
+                csv_path: csv_path,
+                import_directory: Rails.root.join("config", "locales"),
+                multiple_files_per_language: true)
+      end
+
+      expect(importer_instance).to have_received(:import).exactly(import_paths.count)
     end
+  end
+
+  describe "translation:add_missing", type: :task do
+    let(:task) { Rake::Task["translation:add_missing"] }
+    let!(:cleaner_instance) { stub_cleaner }
 
-    it 'triggers i18n task and allows to receive the right arguments' do
+    before do
       allow(I18n::Tasks::CLI).to receive(:start)
+    end
+
+    it "triggers Cleaner and allows to receive the right arguments" do
       task.execute
-      expect(I18n::Tasks::CLI).to have_received(:start).with(["add-missing", "--nil-value"])
+      expect(RailsTranslationManager::Cleaner)
+        .to have_received(:new)
+        .with(Rails.root.join("config", "locales"))
+      expect(cleaner_instance).to have_received(:clean)
+    end
+
+    it "triggers i18n task and allows to receive the right arguments" do
+      task.execute(locale: "fr")
+      expect(I18n::Tasks::CLI).to have_received(:start).with(
+        ["add-missing", "--nil-value", ["-l", "fr"]]
+      )
     end
   end
 
-  describe 'translation:normalize', type: :task do
+  describe "translation:normalize", type: :task do
     let(:task) { Rake::Task["translation:normalize"] }
-  
-    it 'is executed' do
-      expect { task.execute }.to_not output.to_stdout
-    end
+    let!(:cleaner_instance) { stub_cleaner }
 
-    it 'triggers i18n task and allows to receive the right arguments' do
+    before do
       allow(I18n::Tasks::CLI).to receive(:start)
+    end
+
+    it "triggers Cleaner and allows to receive the right arguments" do
+      task.execute(locale_directory: "config/locales")
+      expect(RailsTranslationManager::Cleaner)
+        .to have_received(:new)
+        .with(Rails.root.join("config", "locales"))
+      expect(cleaner_instance).to have_received(:clean)
+    end
+
+    it "triggers i18n task and allows to receive the right arguments" do
       task.execute
       expect(I18n::Tasks::CLI).to have_received(:start).with(["normalize"])
     end
   end
+
+  def stub_importer
+    importer_instance = instance_double(RailsTranslationManager::Importer)
+    allow(RailsTranslationManager::Importer).to receive(:new)
+      .and_return(importer_instance)
+    allow(importer_instance).to receive(:import)
+
+    importer_instance
+  end
+
+  def stub_cleaner
+    cleaner_instance = instance_double(RailsTranslationManager::Cleaner)
+    allow(RailsTranslationManager::Cleaner)
+      .to receive(:new)
+      .and_return(cleaner_instance)
+    allow(cleaner_instance).to receive(:clean)
+
+    cleaner_instance
+  end
 end

data/test/rails_translation_manager/yaml_writer_test.rb

@@ -1,3 +1,5 @@
+require "test_helper"
+require "rails_translation_manager/yaml_writer"
 module RailsTranslationManager
   class DummyWriter
     include YAMLWriter

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: rails_translation_manager
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.3
 platform: ruby
 authors:
 - Edd Sowden
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-12 00:00:00.000000000 Z
+date: 2021-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rails-i18n
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,7 +25,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: i18n-tasks
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: i18n-tasks
+  name: rails-i18n
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -143,6 +157,7 @@ files:
 - lib/rails_translation_manager.rb
 - lib/rails_translation_manager/cleaner.rb
 - lib/rails_translation_manager/exporter.rb
+- lib/rails_translation_manager/i18n_tasks_option_parser.rb
 - lib/rails_translation_manager/importer.rb
 - lib/rails_translation_manager/locale_checker.rb
 - lib/rails_translation_manager/locale_checker/all_locales.rb
@@ -153,20 +168,20 @@ files:
 - lib/rails_translation_manager/locale_checker/missing_foreign_locales.rb
 - lib/rails_translation_manager/locale_checker/plural_forms.rb
 - lib/rails_translation_manager/railtie.rb
-- lib/rails_translation_manager/stealer.rb
-- lib/rails_translation_manager/validator.rb
 - lib/rails_translation_manager/version.rb
 - lib/rails_translation_manager/yaml_writer.rb
 - lib/tasks/translation.rake
-- lib/tasks/translation_helper.rb
 - rails_translation_manager.gemspec
 - spec/locales/cleaner/clean.yml
 - spec/locales/cleaner/with_whitespace.yml
+- spec/locales/importer/fr.csv
+- spec/locales/importer/hy_with_byte_order_mark.csv
 - spec/locales/in_sync/cy/browse.yml
 - spec/locales/in_sync/en/browse.yml
 - spec/locales/out_of_sync/cy.yml
 - spec/locales/out_of_sync/en.yml
 - spec/rails_translation_manager/cleaner_spec.rb
+- spec/rails_translation_manager/importer_spec.rb
 - spec/rails_translation_manager/locale_checker/all_locales_spec.rb
 - spec/rails_translation_manager/locale_checker/incompatible_plurals_spec.rb
 - spec/rails_translation_manager/locale_checker/locale_checker_helper_spec.rb
@@ -178,9 +193,6 @@ files:
 - spec/support/tasks.rb
 - spec/tasks/translation_spec.rb
 - test/rails_translation_manager/exporter_test.rb
-- test/rails_translation_manager/importer_test.rb
-- test/rails_translation_manager/stealer_test.rb
-- test/rails_translation_manager/validator_test.rb
 - test/rails_translation_manager/yaml_writer_test.rb
 - test/test_helper.rb
 - tmp/.gitkeep
@@ -210,11 +222,14 @@ summary: Tasks to manage translation files
 test_files:
 - spec/locales/cleaner/clean.yml
 - spec/locales/cleaner/with_whitespace.yml
+- spec/locales/importer/fr.csv
+- spec/locales/importer/hy_with_byte_order_mark.csv
 - spec/locales/in_sync/cy/browse.yml
 - spec/locales/in_sync/en/browse.yml
 - spec/locales/out_of_sync/cy.yml
 - spec/locales/out_of_sync/en.yml
 - spec/rails_translation_manager/cleaner_spec.rb
+- spec/rails_translation_manager/importer_spec.rb
 - spec/rails_translation_manager/locale_checker/all_locales_spec.rb
 - spec/rails_translation_manager/locale_checker/incompatible_plurals_spec.rb
 - spec/rails_translation_manager/locale_checker/locale_checker_helper_spec.rb
@@ -226,8 +241,5 @@ test_files:
 - spec/support/tasks.rb
 - spec/tasks/translation_spec.rb
 - test/rails_translation_manager/exporter_test.rb
-- test/rails_translation_manager/importer_test.rb
-- test/rails_translation_manager/stealer_test.rb
-- test/rails_translation_manager/validator_test.rb
 - test/rails_translation_manager/yaml_writer_test.rb
 - test/test_helper.rb

data/lib/rails_translation_manager/stealer.rb

@@ -1,85 +0,0 @@
-require "yaml"
-require "i18n"
-require_relative "yaml_writer"
-
-class RailsTranslationManager::Stealer
-  include YAMLWriter
-
-  # locale is the locale name as a string.
-  # source_app_path is the path to the root of the app to steal from.
-  # mapping_file_path is the path to a YAML file mapping translation keys in
-  # the source app to those in the target app. For example:
-  #     document.type: content_item.format
-  #     document.published: content_item.metadata.published
-  # which will import everything under "document.type" and "document.published" 
-  # in the source app, and write it to "content_item.format" and 
-  # "content_item.metadata.published" in the target app.
-  # locales_path is the path to the locale files to output, which is usually
-  # Rails.root.join('config/locales').
-  # The process will preserve data already in the output file if it is not
-  # referenced in the mapping, but will always override data belonging to keys
-  # that are in the mapping.
-  def initialize(locale, source_app_path, mapping_file_path, locales_path)
-    @locale = locale
-    @source_app_path = source_app_path
-    @mapping_file_path = mapping_file_path
-    @locales_path = locales_path
-  end
-
-  def steal_locale
-    target_data = convert_locale(get_target_data)
-    write_yaml(target_locale_path, target_data)
-  end
-
-  def convert_locale(target_data)
-    mapping_data.each do |source, target|
-      data = source_data[@locale]
-      source.split('.').each { |key| data = data.fetch(key, {}) }
-      set_recursive(target_data[@locale], target.split("."), data)
-    end
-    target_data
-  end
-
-  private
-
-  def set_recursive(hash, keys, data)
-    if keys.empty?
-      data
-    else
-      key = keys.shift
-      hash.tap do |h|
-        h.merge!({ key => set_recursive(hash.fetch(key, {}), keys, data)})
-      end
-    end
-  end
-
-  def source_locale_path
-    File.join(@source_app_path, 'config', 'locales', "#{@locale}.yml")
-  end
-
-  def source_data
-    @source_data ||= YAML.load_file(source_locale_path)
-  end
-
-  def target_locale_path
-    File.join(@locales_path, "#{@locale}.yml")
-  end
-
-  def default_target_data
-    { @locale => {} }
-  end
-
-  def get_target_data
-    if File.exist?(target_locale_path)
-      YAML.load_file(target_locale_path) || default_target_data
-    else
-      default_target_data
-    end
-  end
-
-  def mapping_data
-    @mapping_data ||= YAML.load_file(@mapping_file_path)
-  end
-
-end
-

data/lib/rails_translation_manager/validator.rb

@@ -1,92 +0,0 @@
-require 'yaml'
-require 'logger'
-
-class RailsTranslationManager::Validator
-  def initialize(translation_file_path, logger = Logger.new(nil))
-    @translation_file_path = translation_file_path
-    @logger = logger
-  end
-
-  def check!
-    @logger.info "Checking translation files in '#{@translation_file_path}' for unexpected interpolation keys"
-    @logger.info "Loading reference file (#{reference_file_name})"
-    @logger.info "Checking..."
-    reference = load_translation_file("#{@translation_file_path}/#{reference_file_name}")
-    Dir["#{@translation_file_path}/*.yml"].reject do |entry|
-      File.basename(entry) == reference_file_name
-    end.inject([]) do |errors, entry|
-      translation_file = load_translation_file(entry)
-      errors + unexpected_substitution_keys(reference, translation_file)
-    end
-  end
-
-  def unexpected_substitution_keys(reference, translation_file)
-    reference_substitutions = substitutions_in(reference)
-    target_substitutions = substitutions_in(translation_file)
-
-    targets_by_path = target_substitutions.each_with_object({}) do |target, hash|
-      hash[exclude_locale_from_path(target.path)] = target
-    end
-
-    reference_substitutions.each_with_object([]) do |reference, unexpected_substitutions|
-      target = targets_by_path[exclude_locale_from_path(reference.path)]
-      next if target.nil? || reference.has_all_substitutions?(target)
-      unexpected_substitutions << UnexpectedSubstition.new(target, reference)
-    end
-  end
-
-  def substitutions_in(translation_file)
-    flatten(translation_file).reject do |translation|
-      translation.substitutions.empty?
-    end
-  end
-
-  class TranslationEntry < Struct.new(:path, :value)
-    def substitutions
-      @substitutions ||= self.value.scan(/%{([^}]*)}/)
-    end
-
-    def has_all_substitutions?(other)
-      (other.substitutions - self.substitutions).empty?
-    end
-  end
-
-  class UnexpectedSubstition < Struct.new(:target, :reference)
-    def to_s
-      missing = (self.reference.substitutions - self.target.substitutions)
-      extras = (self.target.substitutions - self.reference.substitutions)
-      message = %Q{Key "#{target.path.join('.')}":}
-      if extras.any?
-        message << %Q{ Extra substitutions: ["#{extras.join('", "')}"].}
-      end
-      if missing.any?
-        message << %Q{ Missing substitutions: ["#{missing.join('", "')}"].}
-      end
-      message
-    end
-  end
-
-  def flatten(translation_file, path = [])
-    translation_file.map do |key, value|
-      case value
-      when Hash
-        flatten(value, path + [key])
-      else
-        TranslationEntry.new(path + [key], value || "")
-      end
-    end.flatten
-  end
-
-  def load_translation_file(filename)
-    YAML.load_file(filename)
-  end
-
-  def reference_file_name
-    "en.yml"
-  end
-
-private
-  def exclude_locale_from_path(path)
-    path[1..-1]
-  end
-end

data/test/rails_translation_manager/importer_test.rb

@@ -1,132 +0,0 @@
-require "test_helper"
-require "rails_translation_manager/importer"
-require "tmpdir"
-require "csv"
-
-module RailsTranslationManager
-  class ImporterTest < Minitest::Test
-    test 'should create a new locale file for a filled in translation csv file' do
-      given_csv(:fr,
-        [:key, :source, :translation],
-        ["world_location.type.country", "Country", "Pays"],
-        ["world_location.country", "Germany", "Allemange"],
-        ["other.nested.key", "original", "translated"]
-      )
-
-      Importer.new(:fr, csv_path(:fr), import_directory).import
-
-      yaml_translation_data = YAML.load_file(File.join(import_directory, "fr.yml"))
-      expected = {"fr" => {
-        "world_location" => {
-          "country" => "Allemange",
-          "type" => {
-            "country" => "Pays"
-          }
-        },
-        "other" => {
-          "nested" => {
-            "key" => "translated"
-          }
-        }
-      }}
-      assert_equal expected, yaml_translation_data
-    end
-
-    test 'imports arrays from CSV as arrays' do
-      given_csv(:fr,
-        [:key, :source, :translation],
-        ["fruit", ["Apples", "Bananas", "Pears"], ["Pommes", "Bananes", "Poires"]]
-      )
-
-      Importer.new(:fr, csv_path(:fr), import_directory).import
-
-      yaml_translation_data = YAML.load_file(File.join(import_directory, "fr.yml"))
-      expected = {"fr" => {
-        "fruit" => ["Pommes", "Bananes", "Poires"]
-      }}
-      assert_equal expected, yaml_translation_data
-    end
-
-    test 'interprets string "nil" as nil' do
-      given_csv(:fr,
-        [:key, :source, :translation],
-        ["things", ["one", nil, "two"], ["une", nil, "deux"]]
-      )
-
-      Importer.new(:fr, csv_path(:fr), import_directory).import
-
-      yaml_translation_data = YAML.load_file(File.join(import_directory, "fr.yml"))
-      expected = {"fr" => {
-        "things" => ["une", nil, "deux"]
-      }}
-      assert_equal expected, yaml_translation_data
-    end
-
-    test 'interprets string ":thing" as symbol' do
-      given_csv(:fr,
-        [:key, :source, :translation],
-        ["sentiment", ":whatever", ":bof"]
-      )
-
-      Importer.new(:fr, csv_path(:fr), import_directory).import
-
-      yaml_translation_data = YAML.load_file(File.join(import_directory, "fr.yml"))
-      expected = {"fr" => {
-        "sentiment" => :bof
-      }}
-      assert_equal expected, yaml_translation_data
-    end
-
-    test 'interprets integer strings as integers' do
-      given_csv(:fr,
-        [:key, :source, :translation],
-        ["price", "123", "123"]
-      )
-
-      Importer.new(:fr, csv_path(:fr), import_directory).import
-
-      yaml_translation_data = YAML.load_file(File.join(import_directory, "fr.yml"))
-      expected = {"fr" => {
-        "price" => 123
-      }}
-      assert_equal expected, yaml_translation_data
-    end
-
-    test 'interprets boolean values as booleans, not strings' do
-      given_csv(:fr,
-        [:key, :source, :translation],
-        ["key1", "is true", "true"],
-        ["key2", "is false", "false"]
-      )
-
-      Importer.new(:fr, csv_path(:fr), import_directory).import
-
-      yaml_translation_data = YAML.load_file(File.join(import_directory, "fr.yml"))
-      expected = {"fr" => {
-        "key1" => true,
-        "key2" => false
-      }}
-      assert_equal expected, yaml_translation_data
-    end
-
-  private
-
-    def csv_path(locale)
-      File.join(import_directory, "#{locale}.csv")
-    end
-
-    def given_csv(locale, header_row, *rows)
-      csv = CSV.generate do |csv|
-        csv << CSV::Row.new(["key", "source", "translation"], ["key", "source", "translation"], true)
-        rows.each do |row|
-          csv << CSV::Row.new(["key", "source", "translation"], row)
-        end
-      end
-      File.open(csv_path(locale), "w") { |f| f.write csv.to_s }
-    end
-
-    def import_directory
-      @import_directory ||= Dir.mktmpdir
-    end
-  end
-end

data/test/rails_translation_manager/stealer_test.rb

@@ -1,251 +0,0 @@
-require "test_helper"
-
-require "rails_translation_manager/stealer"
-require "fileutils"
-require "tmpdir"
-require "csv"
-require "i18n"
-
-module RailsTranslationManager
-  class StealerTest < Minitest::Test
-
-    test "converts subtree of items to the same depth" do
-
-      original = {
-        "fr" => {
-          "document" => {
-            "type" => {
-              "type1" => 'premier genre',
-              "type2" => 'deuxième genre',
-              "type3" => 'troisième genre'
-            }
-          }
-        }
-      }
-
-      conversion_mapping = {
-        "document.type" => "content_item.format",
-      }
-
-      expected = {
-        "fr" => {
-          "content_item" => {
-            "format" => {
-              "type1" => 'premier genre',
-              "type2" => 'deuxième genre',
-              "type3" => 'troisième genre'
-            }
-          }
-        }
-      }
-      stealer_test(original, conversion_mapping, expected)
-    end
-
-    test "converts a subtree of items to a different depth" do
-      original = {
-        "fr" => {
-          "document" => {
-            "published" => 'publiée',
-          }
-        }
-      }
-      conversion_mapping = {
-         "document.published" => "content_item.metadata.published"
-      }
-
-      expected = {
-        "fr" => {
-          "content_item" => {
-            "metadata" => {
-              "published" => 'publiée'
-            }
-          }
-        }
-      }
-
-      stealer_test(original, conversion_mapping, expected)
-    end
-
-    test "combines multiple mappings" do
-      original = {
-        "fr" => {
-          "document" => {
-            "type" => {
-              "type1" => 'premier genre',
-            },
-            "published" => 'publiée',
-          }
-        }
-      }
-
-      conversion_mapping = {
-        "document.type" => "content_item.format",
-        "document.published" => "content_item.metadata.published"
-      }
-      expected = {
-        "fr" => {
-          "content_item" => {
-            "format" => {
-              "type1" => 'premier genre',
-            },
-            "metadata" => {
-              "published" => 'publiée',
-            }
-          }
-        }
-      }
-      stealer_test(original, conversion_mapping, expected)
-    end
-
-    test "does not copy over keys with no mapping" do
-      original = {
-        "fr" => {
-          "document" => {
-            "published" => 'publiée',
-            "do_not_want" => 'non voulu'
-          }
-        }
-      }
-      conversion_mapping = {
-         "document.published" => "content_item.metadata.published"
-      }
-
-      expected = {
-        "fr" => {
-          "content_item" => {
-            "metadata" => {
-              "published" => 'publiée'
-            }
-          }
-        }
-      }
-
-      stealer_test(original, conversion_mapping, expected)
-    end
-
-    test "overrides existing translations present in mapping" do
-      original = {
-        "fr" => {
-          "document" => {
-            "published" => 'publiée',
-            "updated" => 'mise au jour',
-          }
-        }
-      }
-
-      conversion_mapping = {
-        "document.published" => "content_item.metadata.published",
-        "document.updated" => "content_item.metadata.updated"
-      }
-
-      existing = {
-        "fr" => {
-          "content_item" => {
-            "metadata" => {
-              "updated" => 'mauvaise traduction'
-            }
-          }
-        }
-      }
-
-      expected = {
-        "fr" => {
-          "content_item" => {
-            "metadata" => {
-              "published" => 'publiée',
-              "updated" => 'mise au jour',
-            }
-          }
-        }
-      }
-      stealer_test(original, conversion_mapping, expected, existing)
-    end
-
-    test "does not override existing translations not in mapping" do
-      original = {
-        "fr" => {
-          "document" => {
-            "published" => 'publiée',
-          }
-        }
-      }
-
-      conversion_mapping = {
-        "document.published" => "content_item.metadata.published"
-      }
-
-      existing = {
-        "fr" => {
-          "content_item" => {
-            "metadata" => {
-              "updated" => 'mise au jour',
-            }
-          }
-        }
-      }
-
-      expected = {
-        "fr" => {
-          "content_item" => {
-            "metadata" => {
-              "published" => 'publiée',
-              "updated" => 'mise au jour',
-            }
-          }
-        }
-      }
-      stealer_test(original, conversion_mapping, expected, existing)
-    end
-
-  private
-
-    def stealer_test(original, mapping, expected, existing=nil)
-      write_source_data(original)
-      write_converter_data(mapping)
-
-      if existing.present?
-        File.open(locale_file, 'w') do |f|
-          f.puts(existing.to_yaml)
-        end
-      end
-
-      stealer = RailsTranslationManager::Stealer.new("fr", source_dir, converter_path, locales_dir)
-      stealer.steal_locale
-
-      assert_equal expected, YAML.load_file(locale_file)
-    end
-
-    def source_dir
-      @source_dir ||= Dir.mktmpdir
-    end
-
-    def source_locale_path
-      File.join(source_dir, 'config/locales')
-    end
-
-    def write_source_data(data)
-      FileUtils.mkdir_p(source_locale_path)
-      File.open(File.join(source_locale_path, 'fr.yml'), 'w') do |f|
-        f.puts(data.to_yaml)
-      end
-    end
-
-    def write_converter_data(data)
-      File.open(converter_path, 'w') do |f|
-        f.puts(data.to_yaml)
-      end
-    end
-
-    def converter_path
-      @converter_path ||= Tempfile.new('fr').path
-    end
-
-    def locales_dir
-      @locales_dir ||= Dir.mktmpdir
-    end
-
-    def locale_file
-      File.join(locales_dir, 'fr.yml')
-    end
-  end
-end

data/test/rails_translation_manager/validator_test.rb

@@ -1,51 +0,0 @@
-# encoding: utf-8
-require 'test_helper'
-require 'rails_translation_manager/validator'
-require 'tmpdir'
-require 'fileutils'
-
-module RailsTranslationManager
-  class ValidatorTest < Minitest::Test
-    def setup
-      @translation_path = Dir.mktmpdir
-      @translation_validator = Validator.new(@translation_path)
-    end
-
-    def teardown
-      FileUtils.remove_entry_secure(@translation_path)
-    end
-
-    def create_translation_file(locale, content)
-      File.open(File.join(@translation_path, "#{locale}.yml"), "w") do |f|
-        f.write(content.lstrip)
-      end
-    end
-
-    test "can create a flattened list of substitutions" do
-      translation_file = YAML.load(%q{
-en:
-  view: View '%{title}'
-  test: foo
-})
-      expected = [Validator::TranslationEntry.new(%w{en view}, "View '%{title}'")]
-      assert_equal expected, @translation_validator.substitutions_in(translation_file)
-    end
-
-    test "detects extra substitution keys" do
-      create_translation_file("en", %q{
-en:
-  document:
-    view: View '%{title}'
-})
-      create_translation_file("sr", %q{
-sr:
-  document:
-    view: Pročitajte '%{naslov}'
-})
-      errors = Validator.new(@translation_path).check!
-
-      expected = %q{Key "sr.document.view": Extra substitutions: ["naslov"]. Missing substitutions: ["title"].}
-      assert_equal [expected], errors.map(&:to_s)
-    end
-  end
-end