i18n-migrations 1.2.3 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13e2d9e3d4aa1ec30fa0584c624d96388fac4aa31e11a6705fa61c39f8ff9a6c
-  data.tar.gz: a88a38c024f92727aaf57bff6950ae5a9d3665d9b86cd26a382cecdb13ec085e
+  metadata.gz: 81cdab3c95ecdf3eadada65b83f94e51733cca841bda8862def4150f3ca182ce
+  data.tar.gz: bcce6ac35266718cc3e49ec7289cba1dedd8f0e012c916a82c1fe323390034a7
 SHA512:
-  metadata.gz: e9eec7ad2d69e380a74b03b26e95be54c9bfb5d59f69cc1115f457f0fbecd0c6f0cec334d2f92b8970550cf42e2e0e9a46818d6b69a2bb7bfbb2f7bd086320ea
-  data.tar.gz: be7bac66cd9be3219e930e9568762c1ffe07b313d7712ab4f7ce424d85625c4770b35bbf749d69dd528dd656c29ab1b615f1cd337ef612d9e32221ec2d3716dc
+  metadata.gz: 65f1fd2c2b047d76b74f6ca85699d8ee5e618b9d35a8e4d24d19a42c851833cfd4cc4f22668f31aecc9eaa2806fffa4801e262ac6193944497fadc48c3c4dcc3
+  data.tar.gz: e041b073626e8546226677bfb927a85a6c3235a233efe33a992b6e7dcb1da2f67c02469552b328b09e6b06ca642fd8579ec378fa08d605a5086342ec75cce5b6

data/bin/i18n-migrate

@@ -10,59 +10,60 @@ def extract_option(name)
   !!ARGV.delete(name)
 end
 
-case ARGV.shift
-when 'setup'
-  puts 'Where should we create a default config file? [.]'
-  dir = gets.chomp
-  dir = dir == '' ? '.' : dir
-  file = I18n::Migrations::Config.copy_default_config_file(dir)
-
-  puts 'You will need to configure this file before you can get going.'
-  puts File.expand_path(file)
-
-when 'new'
-  name = ARGV.shift
-  if name
-    migrator.new_migration name
-  else
-    STDERR.puts 'Usage: im new [name]'
-    exit 1
-  end
-
-when 'migrate'
-  migrator.migrate(ARGV[0] || 'all')
-
-when 'rollback'
-  migrator.rollback(ARGV[0] || 'all')
-
-when 'redo'
-  migrator.rollback(ARGV[0] || 'all')
-  migrator.migrate(ARGV[0] || 'all')
-
-when 'pull'
-  migrator.pull(ARGV[0] || 'all')
+begin
+  case ARGV.shift
+  when 'setup'
+    puts 'Where should we create a default config file? [.]'
+    dir = gets.chomp
+    dir = dir == '' ? '.' : dir
+    file = I18n::Migrations::Config.copy_default_config_file(dir)
+
+    puts 'You will need to configure this file before you can get going.'
+    puts File.expand_path(file)
+
+  when 'new'
+    name = ARGV.shift
+    if name
+      migrator.new_migration name
+    else
+      STDERR.puts 'Usage: im new [name]'
+      exit 1
+    end
+
+  when 'migrate'
+    migrator.migrate(ARGV[0] || 'all')
+
+  when 'rollback'
+    migrator.rollback(ARGV[0] || 'all')
+
+  when 'redo'
+    migrator.rollback(ARGV[0] || 'all')
+    migrator.migrate(ARGV[0] || 'all')
+
+  when 'pull'
+    migrator.pull(ARGV[0] || 'all')
+
+  when 'push'
+    force = extract_option('-f')
+    migrator.push(ARGV[0] || 'all', force)
+
+  when 'validate'
+    migrator.validate(ARGV[0] || 'all')
+
+  when 'new_locale'
+    locale = ARGV.shift
+    if locale
+      migrator.new_locale(locale)
+    else
+      STDERR.puts 'Usage: im new_locale [name]'
+      exit 1
+    end
+
+  when 'version'
+    migrator.version
 
-when 'push'
-  force = extract_option('-f')
-  migrator.push(ARGV[0] || 'all', force)
-
-when 'validate'
-  migrator.validate(ARGV[0] || 'all')
-
-when 'new_locale'
-  locale = ARGV.shift
-  if locale
-    migrator.new_locale(locale)
   else
-    STDERR.puts 'Usage: im new_locale [name]'
-    exit 1
-  end
-
-when 'version'
-  migrator.version
-
-else
-  puts <<-USAGE
+    puts <<-USAGE
 Usage: i18n-migrate [command]
 
 Commands:
@@ -78,5 +79,9 @@ Commands:
   version - Print version of locales.
 
 i18n-migrations version #{I18n::Migrations::VERSION}
-  USAGE
+    USAGE
+  end
+rescue Exception
+  puts "Error\n#{$!.message}".red
+  exit 1
 end

data/lib/i18n/migrations/backends/crowd_translate_backend.rb

@@ -6,8 +6,8 @@ module I18n
       class CrowdTranslateBackend
         attr_reader :client
 
-        def initialize
-          @client = CrowdTranslateClient.new
+        def initialize(client: CrowdTranslateClient.new)
+          @client = client
         end
 
         def pull(locale)
@@ -15,22 +15,49 @@ module I18n
           locale.migrate!
         end
 
-        def push(locale, force = false)
-          raise "CrowdTranslate does not support -f flag yet" if force
-
-          # do this just once
-          unless @migrations_synced
-            @client.sync_migrations(locale.migrations)
-            @migrations_synced = true
+        def push(locale, force: false)
+          if force
+            force_push(locale)
           end
+
           pull(locale)
         end
 
+        # this will replace everything about a locale, it will create a locale that does not yet exist
+        def force_push(locale)
+          data, metadata = locale.read_data_and_metadata(parse: false)
+          client.put_locale(locale.name,
+                            name: locale.name,
+                            yaml_file: data,
+                            yaml_metadata_file: metadata)
+
+        end
+
         def pull_from_crowd_translate(locale)
-          data = @client.get_locale_file(locale.name)
-          locale.write_raw_data("#{locale.name}.yml", data)
+          data = client.get_locale_file(locale.name)
+          locale.data_file.write(data)
           locale.write_remote_version(YAML::load(data)[locale.name])
         end
+
+        def sync_migrations(migrations)
+          local_versions = migrations.all_versions
+          remote_versions = client.get_migration_versions
+
+          if (extra_versions = remote_versions - local_versions).present?
+            raise("You may not upload migrations to the server because it has migrations not found locally: " +
+                    "#{extra_versions.join(', ')}")
+          end
+
+          if (versions_to_add = local_versions - remote_versions).present?
+            versions_to_add.each do |version|
+              begin
+                client.put_migration(version: version, ruby_file: migrations.get_migration(version: version))
+              rescue
+                raise CrowdTranslateError, "... while updating migration: #{migrations.migration_file(version: version)}\n#{$!.message}"
+              end
+            end
+          end
+        end
       end
     end
   end

data/lib/i18n/migrations/backends/crowd_translate_client.rb

@@ -4,58 +4,48 @@ require 'active_support/core_ext/object'
 module I18n
   module Migrations
     module Backends
+      class CrowdTranslateError < StandardError
+      end
+
       class CrowdTranslateClient
         def initialize
           token = ENV['CROWD_TRANSLATE_API_TOKEN']
           raise("You must define CROWD_TRANSLATE_API_TOKEN in order to talk to Crowd Translate") unless token.present?
 
-          server = ENV['CROWD_TRANSLATE_SERVER'] || 'https://crowd-translate.herokuapp.com'
-          @faraday = Faraday.new(
-            url: "#{server}/api/v1",
-            headers: { 'X-CrowdTranslateApiToken' => token },
-          )
+          @server = ENV['CROWD_TRANSLATE_SERVER'] || 'https://crowd-translate.herokuapp.com'
+          @faraday = Faraday.new(url: base_url, headers: { 'X-CrowdTranslateApiToken' => token })
         end
 
-        def sync_migrations(migrations)
-          local_versions = migrations.all_versions
-          remote_versions = JSON.parse(get('migrations.json'))
+        def get_locale_file(locale_code)
+          get("locales/#{locale_code}.yml")
+        end
 
-          if (extra_versions = remote_versions - local_versions).present?
-            raise("You may not upload migrations to the server because it has migrations not found locally: " +
-                    "#{extra_versions.join(', ')}")
-          end
+        def get_migration_versions
+          JSON.parse(get('migrations.json'))
+        end
 
-          if (versions_to_add = local_versions - remote_versions).present?
-            versions_to_add.each do |version|
-              begin
-                put("migrations/#{version}.json",
-                    migration: { ruby_file: migrations.get_migration(version: version) })
-              rescue
-                puts "There was an error updating migration:".red
-                puts "  #{migrations.migration_file(version: version)}".bold
-                raise
-              end
-            end
-          end
+        def put_migration(version:, ruby_file:)
+          put("migrations/#{version}.json", migration: { ruby_file: ruby_file })
         end
 
-        def get_locale_file(locale_code)
-          get("locales/#{locale_code}.yml")
+        def put_locale(locale_code, name:, yaml_file:, yaml_metadata_file:)
+          put("locales/#{locale_code}",
+              locale: { name: name, yaml_file: yaml_file, yaml_metadata_file: yaml_metadata_file })
         end
 
         private
 
         def get(path)
           puts "GET #{path}".bold
-          parse_response @faraday.get path
+          parse_response 'GET', path, @faraday.get(path)
         end
 
         def put(path, params = {})
           puts "PUT #{path} #{params.to_s[0..50]}#{'...' if params.to_s.length > 50}".bold
-          parse_response @faraday.put path, params
+          parse_response 'PUT', path, @faraday.put(path, params)
         end
 
-        def parse_response(response)
+        def parse_response(method, path, response)
           if response.success?
             response.body
           else
@@ -64,9 +54,13 @@ module I18n
                     rescue
                       response.body
                     end
-            raise error
+            raise CrowdTranslateError, "... while calling #{method} #{File.join(base_url, path)}\n#{error}"
           end
         end
+
+        def base_url
+          "#{@server}/api/v1"
+        end
       end
     end
   end

data/lib/i18n/migrations/backends/google_spreadsheets_backend.rb

@@ -1,4 +1,5 @@
 require_relative './google_spreadsheet'
+require_relative '../metadata'
 
 module I18n
   module Migrations
@@ -16,7 +17,7 @@ module I18n
           locale.migrate!
         end
 
-        def push(locale, force = false)
+        def push(locale, force: false)
           return if locale.main_locale?
 
           sheet = get_google_spreadsheet(locale.name)
@@ -33,10 +34,14 @@ module I18n
                                 @config.google_service_account_key_path).sheet
         end
 
+        def sync_migrations(migrations)
+          # nothing to do here in this backend
+        end
+
         def pull_from_sheet(sheet, locale)
           puts "Pulling #{locale.name}"
           data = {}
-          notes = {}
+          metadata = Metadata.new
           count = 0
 
           (2..sheet.num_rows).each do |row|
@@ -44,14 +49,14 @@ module I18n
             if key.present?
               locale.assign_complex_key(data, key.split('.'), value.present? ? value : '')
               if note.present?
-                locale.assign_complex_key(notes, key.split('.'), note)
+                metadata[key] = parse_metadatum(note)
               end
               count += 1
-              print '.'
+              #print '.'
             end
           end
 
-          locale.write_data_and_notes(data, notes)
+          locale.write_data_and_metadata(data, metadata)
           locale.write_remote_version(data)
 
           puts "\n#{count} keys"
@@ -59,7 +64,7 @@ module I18n
 
         def push_to_sheet(sheet, locale)
           main_data = locale.main_locale.read_data
-          data, notes = locale.read_data_and_notes
+          data, metadata = locale.read_data_and_metadata
           row = 2
 
           puts "Pushing #{locale.name}"
@@ -68,9 +73,9 @@ module I18n
             sheet[row, 1] = key
             sheet[row, 2] = value
             sheet[row, 3] = data[key]
-            sheet[row, 4] = notes[key]
+            sheet[row, 4] = unparse_metadatum(metadata[key])
             row += 1
-            print '.'
+            #print '.'
           end
 
           sheet.synchronize
@@ -78,6 +83,28 @@ module I18n
 
           puts "\n#{main_data.keys.length} keys"
         end
+
+        def parse_metadatum(text)
+          m = Metadata::Metadatum.new({})
+          m.notes = text.gsub("[autotranslated]") do
+            m.autotranslated = true
+            ''
+          end.gsub(/\[error: ([^\]]+)\]/) do
+            m.errors << $1
+            ''
+          end.strip
+          m
+        end
+
+        def unparse_metadatum(metadatum)
+          string = []
+          string << '[autotranslated]' if metadatum.autotranslated
+          metadatum.errors.each do |error|
+            string << "[error: #{error}]"
+          end
+          string << metadatum.notes unless metadatum.notes.blank?
+          string.join("\n")
+        end
       end
     end
   end

data/lib/i18n/migrations/google_translate_dictionary.rb

@@ -9,7 +9,7 @@ module I18n
       end
 
       # key is provided so we can figure out if this is text or html
-      # returns [translated term, notes]
+      # returns [translated term, errors || []]
       def lookup(term, key: nil)
         return [term, ''] if @from_locale == @to_locale
 
@@ -19,11 +19,7 @@ module I18n
                                     format: format(key),
                                     q: term)
         translated_term = JSON.parse(response.body)['data']['translations'].first['translatedText']
-        translated_term, errors = fix(term, translated_term, key: key)
-        unless errors.empty?
-          STDERR.puts "'#{term}' => '#{translated_term}'\n#{errors.join(', ').red}"
-        end
-        [translated_term, (errors.map { |e| "[error: #{e}]" } + ['[autotranslated]']).join("\n")]
+        fix(term, translated_term, key: key)
       end
 
       VARIABLE_STRING_REGEX = /%\{[^\}]+\}/
@@ -42,7 +38,7 @@ module I18n
       # returns updated after term, errors
       def fix(before, after, key: nil)
         is_html = format(key) == :html
-        errors = ["#{@to_locale}: #{key}"]
+        errors = []
 
         # do not translate
         @do_not_translate.each do |term, bad_translations|
@@ -114,7 +110,7 @@ module I18n
         end
 
 
-        [after, errors.length > 1 ? errors : []]
+        [after, errors]
       end
 
       private

data/lib/i18n/migrations/locale.rb

@@ -3,6 +3,7 @@ require 'yaml'
 require 'active_support'
 require 'colorize'
 require 'active_support/core_ext/object'
+require_relative './metadata'
 
 # this class does all the work, but doesn't hold config or do more than one locale
 module I18n
@@ -16,7 +17,7 @@ module I18n
           name, locales_dir, main_locale_name, migrations, dictionary
       end
 
-      def validate(data, notes)
+      def validate(data, metadata)
         fix_count, error_count = 0, 0
         main_data = main_locale.read_data
         main_data.each do |key, main_term|
@@ -31,7 +32,7 @@ module I18n
             puts
             fix_count += 1
           end
-          replace_errors_in_notes(notes, key, errors)
+          metadata[key].errors = errors
           if errors.length > 0
             puts "Error #{errors.join(', ').red} #{key.yellow}"
             puts "#{@main_locale_name.bold}: #{main_term}"
@@ -46,18 +47,18 @@ module I18n
       end
 
       def update_info
-        data, notes = read_data_and_notes
-        yield data, notes
-        write_data_and_notes(data, notes)
+        data, metadata = read_data_and_metadata
+        yield data, metadata
+        write_data_and_metadata(data, metadata)
       end
 
       def migrate!
-        update_info do |data, notes|
-          migrate(data, notes)
+        update_info do |data, metadata|
+          migrate(data, metadata)
         end
       end
 
-      def migrate(data, notes)
+      def migrate(data, metadata)
         missing_versions = (@migrations.all_versions - read_versions(data)).sort
         if missing_versions.empty?
           puts "#{@name}: up-to-date"
@@ -65,11 +66,11 @@ module I18n
         end
         puts "#{@name}: Migrating #{missing_versions.join(', ')}"
         missing_versions.each do |version|
-          migrate_to_version(data, notes, version, :up)
+          migrate_to_version(data, metadata, version, :up)
         end
       end
 
-      def rollback(data, notes)
+      def rollback(data, metadata)
         last_version = read_versions(data).last
         if last_version == nil
           puts "#{@name}: no more migrations to roll back"
@@ -78,24 +79,25 @@ module I18n
         puts "#{@name}: Rolling back #{last_version}"
         raise "Can't find #{last_version}.rb to rollback" unless @migrations.all_versions.include?(last_version)
 
-        migrate_to_version(data, notes, last_version, :down)
+        migrate_to_version(data, metadata, last_version, :down)
       end
 
       def create(limit = nil)
-        new_data, new_notes = {}, {}
+        new_data, new_metadata = {}, Metadata.new
         count = 0
         main_data = main_locale.read_data
         main_data.each do |key, term|
           if key == 'VERSION'
             new_data['VERSION'] = main_data['VERSION']
           else
-            new_data[key], new_notes[key] = @dictionary.lookup(term, key: key)
+            new_data[key], errors = @dictionary.lookup(term, key: key)
+            new_metadata[key].errors = errors
           end
           print '.'.green
           break if limit && limit < (count += 1)
         end
         puts
-        write_data_and_notes(new_data, new_notes)
+        write_data_and_metadata(new_data, new_metadata)
       end
 
       def main_locale?
@@ -106,30 +108,28 @@ module I18n
         read_versions(read_data).last
       end
 
-      def read_data
-        read_from_file("#{@name}.yml")
-      end
+      def read_data(parse: true)
+        contents = data_file.read
+        return contents unless parse
 
-      def read_data_and_notes
-        data = read_data
-        notes = main_locale? ? {} : read_from_file("../#{@name}_notes.yml")
-        [data, notes]
+        hash = {}
+        add_to_hash(hash, parse_yaml(contents)[@name.to_s])
+        hash
       end
 
-      def write_data_and_notes(data, notes)
-        write_data(data)
-        write_to_file("../#{@name}_notes.yml", notes) unless main_locale?
+      def read_data_and_metadata(parse: true)
+        data = read_data(parse: parse)
+        metadata = read_metadata(parse: parse)
+        [data, metadata]
       end
 
-      def write_raw_data(filename, data)
-        File.open(File.join(@locales_dir, filename), 'w') do |file|
-          file << data
-        end
+      def write_data_and_metadata(data, metadata)
+        write_data(data)
+        write_metadata(metadata)
       end
 
       def write_remote_version(data)
-        write_to_file("../#{@name}_remote_version.yml",
-                      { 'VERSION' => read_versions(data) })
+        remote_version_file.write({ 'VERSION' => read_versions(data) }.to_yaml)
       end
 
       def main_locale
@@ -151,26 +151,48 @@ module I18n
         end
       end
 
-      private
+      def data_file
+        file("#{name}.yml")
+      end
 
-      def replace_errors_in_notes(all_notes, key, errors)
-        return if all_notes[key].blank? && errors.empty?
+      def metadata_file
+        file("../#{name}_metadata.yml")
+      end
 
-        notes = all_notes[key]
-        notes = notes.present? ? notes.split("\n") : []
-        notes = notes.reject { |n| n.start_with?("[error:") }
-        all_notes[key] = (errors.map { |e| "[error: #{e}]" } + notes).join("\n")
+      def remote_version_file
+        file("../#{@name}_remote_version.yml")
       end
 
+      private
+
       def write_data(data)
-        write_to_file("#{@name}.yml", data)
+        # we have to go from flat keys -> values to a hash that contains other hashes
+        complex_hash = {}
+        data.keys.sort.each do |key|
+          value = data[key]
+          assign_complex_key(complex_hash, key.split('.'), value.present? ? value : '')
+        end
+        data_file.write({ @name => complex_hash }.to_yaml)
+      end
+
+      def read_metadata(parse: true)
+        contents = if metadata_file.exist?
+                     metadata_file.read
+                   else
+                     {}.to_yaml
+                   end
+        parse ? Metadata.new(parse_yaml(contents)) : contents
+      end
+
+      def write_metadata(metadata)
+        metadata_file.write(metadata.to_yaml)
       end
 
-      def migrate_to_version(data, notes, version, direction)
+      def migrate_to_version(data, metadata, version, direction)
         migrations.play_migration(version: version,
                                   locale: @name,
                                   data: data,
-                                  notes: notes,
+                                  metadata: metadata,
                                   dictionary: @dictionary,
                                   direction: direction)
 
@@ -185,26 +207,12 @@ module I18n
         (data['VERSION'] && data['VERSION'].split("\n")) || []
       end
 
-      def read_from_file(filename)
-        filename = File.join(@locales_dir, filename)
-        begin
-          hash = {}
-          add_to_hash(hash, YAML.load(File.read(filename))[@name.to_s])
-          hash
-        rescue
-          puts "Error loading #{filename}"
-          raise
-        end
+      def file(filename)
+        Pathname.new(File.join(@locales_dir, filename))
       end
 
-      def write_to_file(filename, hash)
-        # we have to go from flat keys -> values to a hash that contains other hashes
-        complex_hash = {}
-        hash.keys.sort.each do |key|
-          value = hash[key]
-          assign_complex_key(complex_hash, key.split('.'), value.present? ? value : '')
-        end
-        write_raw_data(filename, { @name => complex_hash }.to_yaml)
+      def parse_yaml(string)
+        YAML.load(string)
       end
 
       # flattens new_hash and adds it to hash

data/lib/i18n/migrations/metadata.rb

@@ -0,0 +1,59 @@
+# this class stores metadata about terms in a locale
+# specifically things like errors, notes, autotranslated
+# it acts kind of like a hash where you give it a key and it returns a metadatum object
+class Metadata
+  def initialize(hash = {})
+    @hash = hash
+  end
+
+  def [](key)
+    if @hash[key].is_a?(Metadatum)
+      @hash[key]
+    else
+      @hash[key] = Metadatum.new(@hash[key])
+    end
+  end
+
+  def []=(key, value)
+    raise("you may only assign a metadatum") unless value.is_a?(Metadatum)
+    @hash[key] = value.dup
+  end
+
+  def delete(key)
+    metadatum = self[key]
+    @hash.delete(key)
+    metadatum
+  end
+
+  def to_h
+    compacted_hash = {}
+    @hash.keys.sort.each do |key|
+      value = @hash[key].to_h
+      compacted_hash[key] = value if value.present?
+    end
+    compacted_hash
+  end
+
+  def to_yaml
+    to_h.to_yaml
+  end
+
+  class Metadatum
+    attr_accessor :errors, :notes, :autotranslated
+
+    def initialize(hash)
+      safe_hash = hash || {}
+      @errors = safe_hash['errors'] || []
+      @notes = safe_hash['notes']
+      @autotranslated = !!safe_hash['autotranslated']
+    end
+
+    def to_h
+      hash = {}
+      hash['errors'] = @errors unless @errors.empty?
+      hash['notes'] = @notes unless @notes.blank?
+      hash['autotranslated'] = @autotranslated if autotranslated
+      hash
+    end
+  end
+end

data/lib/i18n/migrations/migration.rb

@@ -101,10 +101,10 @@ module I18n
 
       def assign_translation(key, term, overrides)
         if overrides[@locale_code.to_sym]
-          @translations.set_term(key, value: overrides[@locale_code.to_sym])
+          @translations.set_term(key, value: overrides[@locale_code.to_sym], errors: [], autotranslated: false)
         else
-          value, notes = @dictionary.lookup(term, key: key)
-          @translations.set_term(key, value: value, notes: notes)
+          value, errors = @dictionary.lookup(term, key: key)
+          @translations.set_term(key, value: value, errors: errors, autotranslated: true)
         end
       end
     end

data/lib/i18n/migrations/migration_factory.rb

@@ -20,14 +20,14 @@ module I18n
         File.join(@migration_dir, "#{version}.rb")
       end
 
-      def play_migration(version:, locale:, data:, notes:, dictionary:, direction:)
+      def play_migration(version:, locale:, data:, metadata:, dictionary:, direction:)
         filename = File.join(@migration_dir, "#{version}.rb")
         require filename
 
         raise("Can't parse version: #{version}") unless version =~ /^(\d{12})_(.*)/
         migration_class_name = "#{$2.camelcase}#{$1}"
 
-        translations = Translations.new(data: data, notes: notes)
+        translations = Translations.new(data: data, metadata: metadata)
         migration = begin
           migration_class_name.constantize.new(locale_code: locale,
                                                translations: translations,
@@ -42,33 +42,37 @@ module I18n
 
       # This is a facade over our translations
       # data = all keys -> all translations in this locale
-      # notes = some keys -> notes about the translation in this locale
+      # metadata = some keys -> metadata about the translation in this locale
       class Translations
-        def initialize(data:, notes:)
-          @data, @notes = data, notes
+        def initialize(data:, metadata:)
+          @data, @metadata = data, metadata
         end
 
         def get_term(key)
           @data[key]
         end
 
-        def set_term(key, value:, notes: nil)
+        def set_term(key, value:, errors:, autotranslated:)
+          #  translated_term, errors = lookup_with_errors(term, key: key)
+          #  unless errors.empty?
+          #    STDERR.puts "'#{term}' => '#{translated_term}'\n#{errors.join(', ').red}"
+          #  end
+          #  [translated_term, (errors.map { |e| "[error: #{e}]" } + ['[autotranslated]']).join("\n")]
+
           @data[key] = value
-          if notes.present?
-            @notes[key] = notes
-          else
-            @notes.delete(key)
-          end
+          @metadata[key].errors = errors
+          @metadata[key].notes = nil
+          @metadata[key].autotranslated = autotranslated
         end
 
         def delete_term(key)
           @data.delete(key)
-          @notes.delete(key)
+          @metadata.delete(key)
         end
 
         def move_term(from, to)
           @data[to] = @data.delete(from)
-          @notes[to] = @notes.delete(from)
+          @metadata[to] = @metadata.delete(from)
         end
       end
     end

data/lib/i18n/migrations/migrator.rb

@@ -64,22 +64,23 @@ end
 
       def rollback(locale_or_all)
         each_locale(locale_or_all) do |locale|
-          locale.update_info do |data, notes|
-            locale.rollback(data, notes)
+          locale.update_info do |data, metadata|
+            locale.rollback(data, metadata)
           end
         end
       end
 
       def pull(locale_or_all)
         each_locale(locale_or_all) do |locale|
-          next if locale.main_locale?
+          # next if locale.main_locale?
           backend.pull(locale)
         end
       end
 
       def push(locale_or_all, force = false)
+        backend.sync_migrations(new_migrations)
         each_locale(locale_or_all, concurrency: config.push_concurrency) do |locale|
-          backend.push(locale, force)
+          backend.push(locale, force: force)
           wait
         end
       end
@@ -97,15 +98,29 @@ end
       def validate(locale_or_all)
         each_locale(locale_or_all, async: false) do |locale|
           next if locale.main_locale?
-          locale.update_info do |data, notes|
-            locale.validate(data, notes)
+          locale.update_info do |data, metadata|
+            locale.validate(data, metadata)
           end
         end
       end
 
+      private def report_locale_on_error(locale, &block)
+        begin
+          block.call locale
+        rescue Backends::CrowdTranslateError
+          puts "Error\n... while working with #{locale.name}\n#{$!.message}".red
+          # we want a readable error for our users, and the error happened in an external system,
+          # so don't give them a stack trace
+        rescue
+          puts "Error\n... while working with #{locale.name}\n#{$!.message}".red
+          raise
+        end
+      end
+
       private def each_locale(name = 'all',
-                              async: config.google_spreadsheet?,
-                              concurrency: config.concurrency)
+                              async: true,
+                              concurrency: config.concurrency,
+                              &block)
         locale_names = name == 'all' ? all_locale_names : [name]
 
         if async
@@ -113,13 +128,13 @@ end
           locale_names.each_slice(concurrency) do |some_locale_names|
             threads = some_locale_names.map do |l|
               locale = locale_for(l)
-              Thread.new { yield locale }
+              Thread.new { report_locale_on_error(locale, &block) }
             end
             threads.each(&:join)
           end
         else
           locale_names.each do |l|
-            yield locale_for(l)
+            report_locale_on_error(locale_for(l), &block)
           end
         end
       end

data/lib/i18n/migrations/version.rb

@@ -1,5 +1,5 @@
 module I18n
   module Migrations
-    VERSION = "1.2.3"
+    VERSION = "2.0.3"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: i18n-migrations
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 2.0.3
 platform: ruby
 authors:
 - Jeremy Lightsmith
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-04 00:00:00.000000000 Z
+date: 2020-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -130,6 +130,7 @@ files:
 - lib/i18n/migrations/config.rb
 - lib/i18n/migrations/google_translate_dictionary.rb
 - lib/i18n/migrations/locale.rb
+- lib/i18n/migrations/metadata.rb
 - lib/i18n/migrations/migration.rb
 - lib/i18n/migrations/migration_factory.rb
 - lib/i18n/migrations/migrator.rb