i18n-migrations 1.2.4 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0b8b7ac35780cf4cf39728ec9c140324a1df5b21868fbeff0d8082eb2b612e1e
-  data.tar.gz: c8c7f5867dc9d8485d527d0b9286a3925eaa69eea40e82ae5ee0a6ca2240f89e
+  metadata.gz: 8fc754b54a85b172209a62717aabb6207bb646be0fca6e04d777732c0c5979bb
+  data.tar.gz: d1b35b6d1d71c7ddfb83904cdff9cca5b307899330b363287c607220ceeccd72
 SHA512:
-  metadata.gz: 3ebecbc35805df21ae0db20ac14931517fb6e55126a3a3b853d1486ab59264f38ea58e66077f665ed226b66afb828544866dae8810fd6c6c93c5ce52f9367d89
-  data.tar.gz: 5a449c05dd1f4d2a2a5492cf0e0ff6fb016344fc1db735e51eaeff4bc9466f62cfb1979c381ad8d35fa503e58d8964fbe044438e9ec1ac51a09fb1483d4356bc
+  metadata.gz: de609cad812e1793e2e9eeff28cfe60ccda2c8b50af2fe97b9f01270f1f661c870b1be3c79bd1ffbe52414e5cb8a54fd2fdc17016eceba94593ca9254bb95d05
+  data.tar.gz: 4f023a9a8b03ffba035c6b293fb51ca68e9274d51cc3adf9a7658575c42df3268ad54e2a04b7ad4d63f43e55968927785f0b045becc646696f3b707eb346849c

data/README.md

@@ -47,20 +47,10 @@ Let's imagine that your config file look like this:
 
 In your project file, you should then have all your english terms in ```config/locales/en.yml```
 
-To create a new locale (like es.yml):
-
-1. Translate all the terms w/ google translate
-
-    > i18n-migrate new_locale es
-    
-2. Create a spreadsheet that is world editable (for now). You'll want to add the link to it to your config file. It should look like:
-
-    | key | en | es | notes | 
-
-2. Push this to your google spreadsheet (the -f means it won't try to pull first)
-
-    > i18n-migrate push -f es
+You can:
 
+* [Create a new locale](https://github.com/transparentclassroom/i18n-migrations/wiki/Create-a-new-locale-(language))
+* [Add a migration](https://github.com/transparentclassroom/i18n-migrations/wiki/Add-a-migration)
 
 ## Development
 
@@ -71,8 +61,9 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 ## Contributing
 
 1. Fork it ( https://github.com/[my-github-username]/i18n-migrations/fork )
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request
+1. Create your feature branch (`git checkout -b my-new-feature`)
+1. Make sure tests are passing (`rake`)
+1. Commit your changes (`git commit -am 'Add some feature'`)
+1. Push to the branch (`git push origin my-new-feature`)
+1. Create a new Pull Request
 

data/bin/i18n-migrate

@@ -10,59 +10,60 @@ def extract_option(name)
   !!ARGV.delete(name)
 end
 
-case ARGV.shift
-when 'setup'
-  puts 'Where should we create a default config file? [.]'
-  dir = gets.chomp
-  dir = dir == '' ? '.' : dir
-  file = I18n::Migrations::Config.copy_default_config_file(dir)
-
-  puts 'You will need to configure this file before you can get going.'
-  puts File.expand_path(file)
-
-when 'new'
-  name = ARGV.shift
-  if name
-    migrator.new_migration name
-  else
-    STDERR.puts 'Usage: im new [name]'
-    exit 1
-  end
-
-when 'migrate'
-  migrator.migrate(ARGV[0] || 'all')
-
-when 'rollback'
-  migrator.rollback(ARGV[0] || 'all')
-
-when 'redo'
-  migrator.rollback(ARGV[0] || 'all')
-  migrator.migrate(ARGV[0] || 'all')
-
-when 'pull'
-  migrator.pull(ARGV[0] || 'all')
+begin
+  case ARGV.shift
+  when 'setup'
+    puts 'Where should we create a default config file? [.]'
+    dir = gets.chomp
+    dir = dir == '' ? '.' : dir
+    file = I18n::Migrations::Config.copy_default_config_file(dir)
+
+    puts 'You will need to configure this file before you can get going.'
+    puts File.expand_path(file)
+
+  when 'new'
+    name = ARGV.shift
+    if name
+      migrator.new_migration name
+    else
+      STDERR.puts 'Usage: im new [name]'
+      exit 1
+    end
+
+  when 'migrate'
+    migrator.migrate(ARGV[0] || 'all')
+
+  when 'rollback'
+    migrator.rollback(ARGV[0] || 'all')
+
+  when 'redo'
+    migrator.rollback(ARGV[0] || 'all')
+    migrator.migrate(ARGV[0] || 'all')
+
+  when 'pull'
+    migrator.pull(ARGV[0] || 'all')
+
+  when 'push'
+    force = extract_option('-f')
+    migrator.push(ARGV[0] || 'all', force)
+
+  when 'validate'
+    migrator.validate(ARGV[0] || 'all')
+
+  when 'new_locale'
+    locale = ARGV.shift
+    if locale
+      migrator.new_locale(locale)
+    else
+      STDERR.puts 'Usage: im new_locale [name]'
+      exit 1
+    end
+
+  when 'version'
+    migrator.version
 
-when 'push'
-  force = extract_option('-f')
-  migrator.push(ARGV[0] || 'all', force)
-
-when 'validate'
-  migrator.validate(ARGV[0] || 'all')
-
-when 'new_locale'
-  locale = ARGV.shift
-  if locale
-    migrator.new_locale(locale)
   else
-    STDERR.puts 'Usage: im new_locale [name]'
-    exit 1
-  end
-
-when 'version'
-  migrator.version
-
-else
-  puts <<-USAGE
+    puts <<-USAGE
 Usage: i18n-migrate [command]
 
 Commands:
@@ -78,5 +79,9 @@ Commands:
   version - Print version of locales.
 
 i18n-migrations version #{I18n::Migrations::VERSION}
-  USAGE
+    USAGE
+  end
+rescue Exception
+  puts "Error\n#{$!.message}".red
+  exit 1
 end

data/lib/i18n/migrations/backends/crowd_translate_backend.rb

@@ -20,27 +20,22 @@ module I18n
             force_push(locale)
           end
 
-          # do this just once
-          unless @migrations_synced
-            sync_migrations(locale.migrations)
-            @migrations_synced = true
-          end
           pull(locale)
         end
 
         # this will replace everything about a locale, it will create a locale that does not yet exist
         def force_push(locale)
-          data, notes = locale.read_data_and_notes(parse: false)
+          data, metadata = locale.read_data_and_metadata(parse: false)
           client.put_locale(locale.name,
                             name: locale.name,
                             yaml_file: data,
-                            yaml_notes_file: notes)
+                            yaml_metadata_file: metadata)
 
         end
 
         def pull_from_crowd_translate(locale)
           data = client.get_locale_file(locale.name)
-          locale.write_raw_data("#{locale.name}.yml", data)
+          locale.data_file.write(data)
           locale.write_remote_version(YAML::load(data)[locale.name])
         end
 
@@ -58,9 +53,7 @@ module I18n
               begin
                 client.put_migration(version: version, ruby_file: migrations.get_migration(version: version))
               rescue
-                puts "There was an error updating migration:".red
-                puts "  #{migrations.migration_file(version: version)}".bold
-                raise
+                raise CrowdTranslateError, "... while updating migration: #{migrations.migration_file(version: version)}\n#{$!.message}"
               end
             end
           end

data/lib/i18n/migrations/backends/crowd_translate_client.rb

@@ -4,16 +4,16 @@ require 'active_support/core_ext/object'
 module I18n
   module Migrations
     module Backends
+      class CrowdTranslateError < StandardError
+      end
+
       class CrowdTranslateClient
         def initialize
           token = ENV['CROWD_TRANSLATE_API_TOKEN']
           raise("You must define CROWD_TRANSLATE_API_TOKEN in order to talk to Crowd Translate") unless token.present?
 
-          server = ENV['CROWD_TRANSLATE_SERVER'] || 'https://crowd-translate.herokuapp.com'
-          @faraday = Faraday.new(
-            url: "#{server}/api/v1",
-            headers: { 'X-CrowdTranslateApiToken' => token },
-          )
+          @server = ENV['CROWD_TRANSLATE_SERVER'] || 'https://crowd-translate.herokuapp.com'
+          @faraday = Faraday.new(url: base_url, headers: { 'X-CrowdTranslateApiToken' => token })
         end
 
         def get_locale_file(locale_code)
@@ -28,35 +28,43 @@ module I18n
           put("migrations/#{version}.json", migration: { ruby_file: ruby_file })
         end
 
-        def put_locale(locale_code, name:, yaml_file:, yaml_notes_file:)
+        def put_locale(locale_code, name:, yaml_file:, yaml_metadata_file:)
           put("locales/#{locale_code}",
-              locale: { name: name, yaml_file: yaml_file, yaml_notes_file: yaml_notes_file })
+              locale: { name: name, yaml_file: yaml_file, yaml_metadata_file: yaml_metadata_file })
         end
 
         private
 
         def get(path)
           puts "GET #{path}".bold
-          parse_response @faraday.get path
+          parse_response 'GET', path, @faraday.get(path)
         end
 
         def put(path, params = {})
           puts "PUT #{path} #{params.to_s[0..50]}#{'...' if params.to_s.length > 50}".bold
-          parse_response @faraday.put path, params
+          parse_response 'PUT', path, @faraday.put(path, params)
         end
 
-        def parse_response(response)
+        def parse_response(method, path, response)
           if response.success?
             response.body
           else
-            error = begin
-                      JSON.parse(response.body)['error']
-                    rescue
-                      response.body
+            error = if response.status == 503
+                      'Server timed out - your request may have succeed, check the server for more details'
+                    else
+                      begin
+                        JSON.parse(response.body)['error']
+                      rescue
+                        response.body
+                      end
                     end
-            raise error
+            raise CrowdTranslateError, "... while calling #{method} #{File.join(base_url, path)}\n#{error}"
           end
         end
+
+        def base_url
+          "#{@server}/api/v1"
+        end
       end
     end
   end

data/lib/i18n/migrations/backends/google_spreadsheets_backend.rb

@@ -1,4 +1,5 @@
 require_relative './google_spreadsheet'
+require_relative '../metadata'
 
 module I18n
   module Migrations
@@ -33,10 +34,14 @@ module I18n
                                 @config.google_service_account_key_path).sheet
         end
 
+        def sync_migrations(migrations)
+          # nothing to do here in this backend
+        end
+
         def pull_from_sheet(sheet, locale)
           puts "Pulling #{locale.name}"
           data = {}
-          notes = {}
+          metadata = Metadata.new
           count = 0
 
           (2..sheet.num_rows).each do |row|
@@ -44,14 +49,14 @@ module I18n
             if key.present?
               locale.assign_complex_key(data, key.split('.'), value.present? ? value : '')
               if note.present?
-                locale.assign_complex_key(notes, key.split('.'), note)
+                metadata[key] = parse_metadatum(note)
               end
               count += 1
-              print '.'
+              #print '.'
             end
           end
 
-          locale.write_data_and_notes(data, notes)
+          locale.write_data_and_metadata(data, metadata)
           locale.write_remote_version(data)
 
           puts "\n#{count} keys"
@@ -59,7 +64,7 @@ module I18n
 
         def push_to_sheet(sheet, locale)
           main_data = locale.main_locale.read_data
-          data, notes = locale.read_data_and_notes
+          data, metadata = locale.read_data_and_metadata
           row = 2
 
           puts "Pushing #{locale.name}"
@@ -68,9 +73,9 @@ module I18n
             sheet[row, 1] = key
             sheet[row, 2] = value
             sheet[row, 3] = data[key]
-            sheet[row, 4] = notes[key]
+            sheet[row, 4] = unparse_metadatum(metadata[key])
             row += 1
-            print '.'
+            #print '.'
           end
 
           sheet.synchronize
@@ -78,6 +83,28 @@ module I18n
 
           puts "\n#{main_data.keys.length} keys"
         end
+
+        def parse_metadatum(text)
+          m = Metadata::Metadatum.new({})
+          m.notes = text.gsub("[autotranslated]") do
+            m.autotranslated = true
+            ''
+          end.gsub(/\[error: ([^\]]+)\]/) do
+            m.errors << $1
+            ''
+          end.strip
+          m
+        end
+
+        def unparse_metadatum(metadatum)
+          string = []
+          string << '[autotranslated]' if metadatum.autotranslated
+          metadatum.errors.each do |error|
+            string << "[error: #{error}]"
+          end
+          string << metadatum.notes unless metadatum.notes.blank?
+          string.join("\n")
+        end
       end
     end
   end

data/lib/i18n/migrations/google_translate_dictionary.rb

@@ -9,7 +9,7 @@ module I18n
       end
 
       # key is provided so we can figure out if this is text or html
-      # returns [translated term, notes]
+      # returns [translated term, errors || []]
       def lookup(term, key: nil)
         return [term, ''] if @from_locale == @to_locale
 
@@ -19,11 +19,7 @@ module I18n
                                     format: format(key),
                                     q: term)
         translated_term = JSON.parse(response.body)['data']['translations'].first['translatedText']
-        translated_term, errors = fix(term, translated_term, key: key)
-        unless errors.empty?
-          STDERR.puts "'#{term}' => '#{translated_term}'\n#{errors.join(', ').red}"
-        end
-        [translated_term, (errors.map { |e| "[error: #{e}]" } + ['[autotranslated]']).join("\n")]
+        fix(term, translated_term, key: key)
       end
 
       VARIABLE_STRING_REGEX = /%\{[^\}]+\}/
@@ -42,7 +38,7 @@ module I18n
       # returns updated after term, errors
       def fix(before, after, key: nil)
         is_html = format(key) == :html
-        errors = ["#{@to_locale}: #{key}"]
+        errors = []
 
         # do not translate
         @do_not_translate.each do |term, bad_translations|
@@ -114,7 +110,7 @@ module I18n
         end
 
 
-        [after, errors.length > 1 ? errors : []]
+        [after, errors]
       end
 
       private

data/lib/i18n/migrations/locale.rb

@@ -3,6 +3,7 @@ require 'yaml'
 require 'active_support'
 require 'colorize'
 require 'active_support/core_ext/object'
+require_relative './metadata'
 
 # this class does all the work, but doesn't hold config or do more than one locale
 module I18n
@@ -16,7 +17,7 @@ module I18n
           name, locales_dir, main_locale_name, migrations, dictionary
       end
 
-      def validate(data, notes)
+      def validate(data, metadata)
         fix_count, error_count = 0, 0
         main_data = main_locale.read_data
         main_data.each do |key, main_term|
@@ -31,7 +32,7 @@ module I18n
             puts
             fix_count += 1
           end
-          replace_errors_in_notes(notes, key, errors)
+          metadata[key].errors = errors
           if errors.length > 0
             puts "Error #{errors.join(', ').red} #{key.yellow}"
             puts "#{@main_locale_name.bold}: #{main_term}"
@@ -46,18 +47,18 @@ module I18n
       end
 
       def update_info
-        data, notes = read_data_and_notes
-        yield data, notes
-        write_data_and_notes(data, notes)
+        data, metadata = read_data_and_metadata
+        yield data, metadata
+        write_data_and_metadata(data, metadata)
       end
 
       def migrate!
-        update_info do |data, notes|
-          migrate(data, notes)
+        update_info do |data, metadata|
+          migrate(data, metadata)
         end
       end
 
-      def migrate(data, notes)
+      def migrate(data, metadata)
         missing_versions = (@migrations.all_versions - read_versions(data)).sort
         if missing_versions.empty?
           puts "#{@name}: up-to-date"
@@ -65,11 +66,11 @@ module I18n
         end
         puts "#{@name}: Migrating #{missing_versions.join(', ')}"
         missing_versions.each do |version|
-          migrate_to_version(data, notes, version, :up)
+          migrate_to_version(data, metadata, version, :up)
         end
       end
 
-      def rollback(data, notes)
+      def rollback(data, metadata)
         last_version = read_versions(data).last
         if last_version == nil
           puts "#{@name}: no more migrations to roll back"
@@ -78,24 +79,25 @@ module I18n
         puts "#{@name}: Rolling back #{last_version}"
         raise "Can't find #{last_version}.rb to rollback" unless @migrations.all_versions.include?(last_version)
 
-        migrate_to_version(data, notes, last_version, :down)
+        migrate_to_version(data, metadata, last_version, :down)
       end
 
       def create(limit = nil)
-        new_data, new_notes = {}, {}
+        new_data, new_metadata = {}, Metadata.new
         count = 0
         main_data = main_locale.read_data
         main_data.each do |key, term|
           if key == 'VERSION'
             new_data['VERSION'] = main_data['VERSION']
           else
-            new_data[key], new_notes[key] = @dictionary.lookup(term, key: key)
+            new_data[key], errors = @dictionary.lookup(term, key: key)
+            new_metadata[key].errors = errors
           end
           print '.'.green
           break if limit && limit < (count += 1)
         end
         puts
-        write_data_and_notes(new_data, new_notes)
+        write_data_and_metadata(new_data, new_metadata)
       end
 
       def main_locale?
@@ -107,29 +109,27 @@ module I18n
       end
 
       def read_data(parse: true)
-        read_from_file("#{@name}.yml", parse: parse)
+        contents = data_file.read
+        return contents unless parse
+
+        hash = {}
+        add_to_hash(hash, parse_yaml(contents)[@name.to_s])
+        hash
       end
 
-      def read_data_and_notes(parse: true)
+      def read_data_and_metadata(parse: true)
         data = read_data(parse: parse)
-        notes = main_locale? ? (parse ? {} : '--- {}') : read_from_file("../#{@name}_notes.yml", parse: parse)
-        [data, notes]
+        metadata = read_metadata(parse: parse)
+        [data, metadata]
       end
 
-      def write_data_and_notes(data, notes)
+      def write_data_and_metadata(data, metadata)
         write_data(data)
-        write_to_file("../#{@name}_notes.yml", notes) unless main_locale?
-      end
-
-      def write_raw_data(filename, data)
-        File.open(File.join(@locales_dir, filename), 'w') do |file|
-          file << data
-        end
+        write_metadata(metadata)
       end
 
       def write_remote_version(data)
-        write_to_file("../#{@name}_remote_version.yml",
-                      { 'VERSION' => read_versions(data) })
+        remote_version_file.write({ 'VERSION' => read_versions(data) }.to_yaml)
       end
 
       def main_locale
@@ -151,26 +151,48 @@ module I18n
         end
       end
 
-      private
+      def data_file
+        file("#{name}.yml")
+      end
 
-      def replace_errors_in_notes(all_notes, key, errors)
-        return if all_notes[key].blank? && errors.empty?
+      def metadata_file
+        file("../#{name}_metadata.yml")
+      end
 
-        notes = all_notes[key]
-        notes = notes.present? ? notes.split("\n") : []
-        notes = notes.reject { |n| n.start_with?("[error:") }
-        all_notes[key] = (errors.map { |e| "[error: #{e}]" } + notes).join("\n")
+      def remote_version_file
+        file("../#{@name}_remote_version.yml")
       end
 
+      private
+
       def write_data(data)
-        write_to_file("#{@name}.yml", data)
+        # we have to go from flat keys -> values to a hash that contains other hashes
+        complex_hash = {}
+        data.keys.sort.each do |key|
+          value = data[key]
+          assign_complex_key(complex_hash, key.split('.'), value.present? ? value : '')
+        end
+        data_file.write({ @name => complex_hash }.to_yaml)
+      end
+
+      def read_metadata(parse: true)
+        contents = if metadata_file.exist?
+                     metadata_file.read
+                   else
+                     {}.to_yaml
+                   end
+        parse ? Metadata.new(parse_yaml(contents)) : contents
       end
 
-      def migrate_to_version(data, notes, version, direction)
+      def write_metadata(metadata)
+        metadata_file.write(metadata.to_yaml)
+      end
+
+      def migrate_to_version(data, metadata, version, direction)
         migrations.play_migration(version: version,
                                   locale: @name,
                                   data: data,
-                                  notes: notes,
+                                  metadata: metadata,
                                   dictionary: @dictionary,
                                   direction: direction)
 
@@ -185,29 +207,12 @@ module I18n
         (data['VERSION'] && data['VERSION'].split("\n")) || []
       end
 
-      def read_from_file(filename, parse: true)
-        filename = File.join(@locales_dir, filename)
-        begin
-          contents = File.read(filename)
-          return contents unless parse
-
-          hash = {}
-          add_to_hash(hash, YAML.load(contents)[@name.to_s])
-          hash
-        rescue
-          puts "Error loading #{filename}"
-          raise
-        end
+      def file(filename)
+        Pathname.new(File.join(@locales_dir, filename))
       end
 
-      def write_to_file(filename, hash)
-        # we have to go from flat keys -> values to a hash that contains other hashes
-        complex_hash = {}
-        hash.keys.sort.each do |key|
-          value = hash[key]
-          assign_complex_key(complex_hash, key.split('.'), value.present? ? value : '')
-        end
-        write_raw_data(filename, { @name => complex_hash }.to_yaml)
+      def parse_yaml(string)
+        YAML.load(string)
       end
 
       # flattens new_hash and adds it to hash

data/lib/i18n/migrations/metadata.rb

@@ -0,0 +1,59 @@
+# this class stores metadata about terms in a locale
+# specifically things like errors, notes, autotranslated
+# it acts kind of like a hash where you give it a key and it returns a metadatum object
+class Metadata
+  def initialize(hash = {})
+    @hash = hash
+  end
+
+  def [](key)
+    if @hash[key].is_a?(Metadatum)
+      @hash[key]
+    else
+      @hash[key] = Metadatum.new(@hash[key])
+    end
+  end
+
+  def []=(key, value)
+    raise("you may only assign a metadatum") unless value.is_a?(Metadatum)
+    @hash[key] = value.dup
+  end
+
+  def delete(key)
+    metadatum = self[key]
+    @hash.delete(key)
+    metadatum
+  end
+
+  def to_h
+    compacted_hash = {}
+    @hash.keys.sort.each do |key|
+      value = @hash[key].to_h
+      compacted_hash[key] = value if value.present?
+    end
+    compacted_hash
+  end
+
+  def to_yaml
+    to_h.to_yaml
+  end
+
+  class Metadatum
+    attr_accessor :errors, :notes, :autotranslated
+
+    def initialize(hash)
+      safe_hash = hash || {}
+      @errors = safe_hash['errors'] || []
+      @notes = safe_hash['notes']
+      @autotranslated = !!safe_hash['autotranslated']
+    end
+
+    def to_h
+      hash = {}
+      hash['errors'] = @errors unless @errors.empty?
+      hash['notes'] = @notes unless @notes.blank?
+      hash['autotranslated'] = @autotranslated if autotranslated
+      hash
+    end
+  end
+end

data/lib/i18n/migrations/migration.rb

@@ -101,10 +101,10 @@ module I18n
 
       def assign_translation(key, term, overrides)
         if overrides[@locale_code.to_sym]
-          @translations.set_term(key, value: overrides[@locale_code.to_sym])
+          @translations.set_term(key, value: overrides[@locale_code.to_sym], errors: [], autotranslated: false)
         else
-          value, notes = @dictionary.lookup(term, key: key)
-          @translations.set_term(key, value: value, notes: notes)
+          value, errors = @dictionary.lookup(term, key: key)
+          @translations.set_term(key, value: value, errors: errors, autotranslated: true)
         end
       end
     end

data/lib/i18n/migrations/migration_factory.rb

@@ -20,55 +20,70 @@ module I18n
         File.join(@migration_dir, "#{version}.rb")
       end
 
-      def play_migration(version:, locale:, data:, notes:, dictionary:, direction:)
+      def play_migration(version:, locale:, data:, metadata:, dictionary:, direction:)
         filename = File.join(@migration_dir, "#{version}.rb")
         require filename
 
         raise("Can't parse version: #{version}") unless version =~ /^(\d{12})_(.*)/
         migration_class_name = "#{$2.camelcase}#{$1}"
 
-        translations = Translations.new(data: data, notes: notes)
+        translations = Translations.new(locale_code: locale, data: data, metadata: metadata)
         migration = begin
-          migration_class_name.constantize.new(locale_code: locale,
-                                               translations: translations,
-                                               dictionary: dictionary,
-                                               direction: direction)
-        rescue
-          raise "Couldn't load migration #{migration_class_name} in #{filename}"
-        end
+                      migration_class_name.constantize.new(locale_code: locale,
+                                                           translations: translations,
+                                                           dictionary: dictionary,
+                                                           direction: direction)
+                    rescue
+                      raise "Couldn't load migration #{migration_class_name} in #{filename}"
+                    end
 
         migration.change
       end
 
       # This is a facade over our translations
       # data = all keys -> all translations in this locale
-      # notes = some keys -> notes about the translation in this locale
+      # metadata = some keys -> metadata about the translation in this locale
       class Translations
-        def initialize(data:, notes:)
-          @data, @notes = data, notes
+        def initialize(locale_code:, data:, metadata:)
+          @locale_code, @data, @metadata = locale_code, data, metadata
         end
 
         def get_term(key)
           @data[key]
         end
 
-        def set_term(key, value:, notes: nil)
+        def set_term(key, value:, errors:, autotranslated:)
+          #  translated_term, errors = lookup_with_errors(term, key: key)
+          #  unless errors.empty?
+          #    STDERR.puts "'#{term}' => '#{translated_term}'\n#{errors.join(', ').red}"
+          #  end
+          #  [translated_term, (errors.map { |e| "[error: #{e}]" } + ['[autotranslated]']).join("\n")]
+
           @data[key] = value
-          if notes.present?
-            @notes[key] = notes
-          else
-            @notes.delete(key)
+          @metadata[key].errors = errors
+          @metadata[key].notes = nil
+          @metadata[key].autotranslated = autotranslated
+          if errors.present?
+            puts [
+                   "=" * 100,
+                   "#{@locale_code}: #{key}",
+                   value,
+                   errors.map { |e| "Error: #{e}".red }.join("\n"),
+                   "=" * 100,
+                 ].compact.join("\n")
+            # puts ["Error with #{@locale_code}: #{key}: \n  #{errors.join("\n  ")}".red,
+            #       value].compact.join("\n")
           end
         end
 
         def delete_term(key)
           @data.delete(key)
-          @notes.delete(key)
+          @metadata.delete(key)
         end
 
         def move_term(from, to)
           @data[to] = @data.delete(from)
-          @notes[to] = @notes.delete(from)
+          @metadata[to] = @metadata.delete(from)
         end
       end
     end

data/lib/i18n/migrations/migrator.rb

@@ -64,20 +64,21 @@ end
 
       def rollback(locale_or_all)
         each_locale(locale_or_all) do |locale|
-          locale.update_info do |data, notes|
-            locale.rollback(data, notes)
+          locale.update_info do |data, metadata|
+            locale.rollback(data, metadata)
           end
         end
       end
 
       def pull(locale_or_all)
         each_locale(locale_or_all) do |locale|
-          next if locale.main_locale?
+          # next if locale.main_locale?
           backend.pull(locale)
         end
       end
 
       def push(locale_or_all, force = false)
+        backend.sync_migrations(new_migrations)
         each_locale(locale_or_all, concurrency: config.push_concurrency) do |locale|
           backend.push(locale, force: force)
           wait
@@ -97,15 +98,29 @@ end
       def validate(locale_or_all)
         each_locale(locale_or_all, async: false) do |locale|
           next if locale.main_locale?
-          locale.update_info do |data, notes|
-            locale.validate(data, notes)
+          locale.update_info do |data, metadata|
+            locale.validate(data, metadata)
           end
         end
       end
 
+      private def report_locale_on_error(locale, &block)
+        begin
+          block.call locale
+        rescue Backends::CrowdTranslateError
+          puts "Error\n... while working with #{locale.name}\n#{$!.message}".red
+          # we want a readable error for our users, and the error happened in an external system,
+          # so don't give them a stack trace
+        rescue
+          puts "Error\n... while working with #{locale.name}\n#{$!.message}".red
+          raise
+        end
+      end
+
       private def each_locale(name = 'all',
-                              async: config.google_spreadsheet?,
-                              concurrency: config.concurrency)
+                              async: true,
+                              concurrency: config.concurrency,
+                              &block)
         locale_names = name == 'all' ? all_locale_names : [name]
 
         if async
@@ -113,13 +128,13 @@ end
           locale_names.each_slice(concurrency) do |some_locale_names|
             threads = some_locale_names.map do |l|
               locale = locale_for(l)
-              Thread.new { yield locale }
+              Thread.new { report_locale_on_error(locale, &block) }
             end
             threads.each(&:join)
           end
         else
           locale_names.each do |l|
-            yield locale_for(l)
+            report_locale_on_error(locale_for(l), &block)
           end
         end
       end

data/lib/i18n/migrations/version.rb

@@ -1,5 +1,5 @@
 module I18n
   module Migrations
-    VERSION = "1.2.4"
+    VERSION = "2.0.4"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: i18n-migrations
 version: !ruby/object:Gem::Version
-  version: 1.2.4
+  version: 2.0.4
 platform: ruby
 authors:
 - Jeremy Lightsmith
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-04 00:00:00.000000000 Z
+date: 2020-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -130,6 +130,7 @@ files:
 - lib/i18n/migrations/config.rb
 - lib/i18n/migrations/google_translate_dictionary.rb
 - lib/i18n/migrations/locale.rb
+- lib/i18n/migrations/metadata.rb
 - lib/i18n/migrations/migration.rb
 - lib/i18n/migrations/migration_factory.rb
 - lib/i18n/migrations/migrator.rb