lokalise_rails 1.4.0 → 2.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6448a3718185d09a6712428f671e02121e59c967199ae6e24f27b8623f12b280
-  data.tar.gz: c0f2501d9c937b26088ee23a6603a842121cb6a88516ae19d09d40abe1968161
+  metadata.gz: d8c05601cd8f8684c1523aa4b92d87d10afad700c50d197e40b6886fd4bc9412
+  data.tar.gz: ddc3d0616655b4bda4a40ecf1a96af9c17c5ef54d7c372a05bdf9eb29c2dcdee
 SHA512:
-  metadata.gz: be178f379eb4abbb4d50c4f3817422c665bed0c8a75f256c083b0229a57594956ca8c42648fe98be40ed1c4183f0e00fd51fcaf153a87ed408279c1d2ac152ec
-  data.tar.gz: db7dd0efdb0d9a7b72116adba959a04b899e41d5ccd89181f48e2ec2e1d097ebfebb732e6592c725b831cebf89134e77e0639deb8591876267306583064efa96
+  metadata.gz: 1bfb463dbed7826bf9cad245c53cca76b5ec0c0bc35ddeb42c24c0e9789e93d77adf45e12c772dfcb764e60ffbe3724ecbebb0106bbead653030db1a458df275
+  data.tar.gz: aadd4c0108d6871683c90f4d0d8647f80f072a34e0d4597ec9d4fcb97a9e9494e085b6fe93481925d71a594e0a110ef6575d0499ede0e407d517727bdb0b9baf

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 2.0.0.rc1 (12-Aug-21)
+
+* **Lokalise is introducing API rate limiting.** Access to all endpoints will be limited to 6 requests per second from 14 September, 2021. This limit is applied per API token and per IP address. If you exceed the limit, a 429 HTTP status code will be returned and a `Lokalise::Error::TooManyRequests` exception will be raised. Therefore, to overcome this issue LokaliseRails is introducing an exponential backoff mechanism for file exports. Why? Because if you have, say, 10 translation files, we'll have to send 10 separate API requests which will probably mean exceeding the limit (this is not the case with importing as we're always receiving a single archive). Thus, if the HTTP status code 429 was received, we'll do the following:
+
+```ruby
+sleep 2 ** retries
+retries += 1
+```
+
+* If the maximum number of retries has been reached LokaliseRails will re-raise the `Lokalise::Error::TooManyRequests` error and give up. By default, the maximum number of retries is set to `5` but you can control it using the `max_retries_export` option.
+* Enabled compression for the API client.
+
 ## 1.4.0 (29-Jun-21)
 
 * Re-worked exception handling. Now when something goes wrong during the import or export process, this gem will re-raise all such errors (previously it just printed out some errors to the `$stdout`). If you run a Rake task, it will exit with a status code `1` and the actual error message. If you run a task programattically, you'll get an exception.

data/README.md

@@ -134,6 +134,8 @@ en_US:
 c.skip_file_export = ->(file) { f.split[1].to_s.include?('fr') }
 ```
 
+* `max_retries_export` (`integer`) - this option is introduced to properly handle Lokalise API rate limiting during file exporting. If the HTTP status code 429 (too many requests) has been received, LokaliseRails will apply an exponential backoff mechanism with a very simple formula: `2 ** retries` (initially `retries` is `0`). If the maximum number of retries has been reached, a `Lokalise::Error::TooManyRequests` exception will be raised and the export operation will be halted. By default, LokaliseRails will make up to `5` retries which potentially means `1 + 2 + 4 + 8 + 16 + 32 = 63` seconds of waiting time. If the `max_retries_export` is less than `1`, LokaliseRails will not perform any retries and give up immediately after receiving error 429.
+
 ### Settings to work with formats other than YAML
 
 If your translation files are not in YAML format, you will need to adjust the following options:

data/lib/generators/templates/lokalise_rails_config.rb

@@ -16,6 +16,9 @@ LokaliseRails.config do |c|
   # Provide request timeouts for the Lokalise API client:
   # c.timeouts = {open_timeout: nil, timeout: nil}
 
+  # Provide maximum number of retries for file exporting:
+  # c.max_retries_export = 5
+
   # Import options have the following defaults:
   # c.import_opts = {
   #   format: 'yaml',

data/lib/lokalise_rails.rb

@@ -10,7 +10,7 @@ module LokaliseRails
     attr_accessor :api_token, :project_id
     attr_writer :import_opts, :import_safe_mode, :export_opts, :locales_path,
                 :file_ext_regexp, :skip_file_export, :branch, :timeouts,
-                :translations_loader, :translations_converter, :lang_iso_inferer
+                :translations_loader, :translations_converter, :lang_iso_inferer, :max_retries_export
 
     # Main interface to provide configuration options for rake tasks
     def config
@@ -32,6 +32,11 @@ module LokaliseRails
       @timeouts || {}
     end
 
+    # Maximum number of retries for file exporting
+    def max_retries_export
+      @max_retries_export || 5
+    end
+
     # Regular expression used to select translation files with proper extensions
     def file_ext_regexp
       @file_ext_regexp || /\.ya?ml\z/i

data/lib/lokalise_rails/task_definition/base.rb

@@ -13,7 +13,7 @@ module LokaliseRails
         #
         # @return [Lokalise::Client]
         def api_client
-          @api_client ||= ::Lokalise.client LokaliseRails.api_token, LokaliseRails.timeouts
+          @api_client ||= ::Lokalise.client LokaliseRails.api_token, {enable_compression: true}.merge(LokaliseRails.timeouts)
         end
 
         # Resets API client

data/lib/lokalise_rails/task_definition/exporter.rb

@@ -14,9 +14,7 @@ module LokaliseRails
 
           queued_processes = []
           each_file do |full_path, relative_path|
-            queued_processes << api_client.upload_file(
-              project_id_with_branch, opts(full_path, relative_path)
-            )
+            queued_processes << do_upload(full_path, relative_path)
           rescue StandardError => e
             raise e.class, "Error while trying to upload #{full_path}: #{e.message}"
           end
@@ -26,6 +24,21 @@ module LokaliseRails
           queued_processes
         end
 
+        # Performs the actual file uploading to Lokalise. If the API rate limit is exceeed,
+        # applies exponential backoff
+        def do_upload(f_path, r_path)
+          retries = 0
+          begin
+            api_client.upload_file project_id_with_branch, opts(f_path, r_path)
+          rescue Lokalise::Error::TooManyRequests => e
+            raise(e.class, "Gave up after #{retries} retries") if retries >= LokaliseRails.max_retries_export
+
+            sleep 2**retries
+            retries += 1
+            retry
+          end
+        end
+
         # Processes each translation file in the specified directory
         def each_file
           return unless block_given?

data/lib/lokalise_rails/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LokaliseRails
-  VERSION = '1.4.0'
+  VERSION = '2.0.0.rc1'
 end

data/spec/lib/lokalise_rails/task_definition/exporter_spec.rb

@@ -7,6 +7,45 @@ describe LokaliseRails::TaskDefinition::Exporter do
   let(:path) { "#{Rails.root}/config/locales/nested/#{filename}" }
   let(:relative_name) { "nested/#{filename}" }
 
+  context 'with many translation files' do
+    let(:fake_class) { described_class }
+
+    before :all do
+      add_translation_files! with_ru: true, additional: 5
+    end
+
+    after :all do
+      rm_translation_files
+    end
+
+    describe '.export!' do
+      it 'sends a proper API request and handles rate limiting' do
+        allow_project_id '672198945b7d72fc048021.15940510'
+
+        process = VCR.use_cassette('upload_files_multiple') do
+          described_class.export!
+        end.first
+
+        expect(process.project_id).to eq(LokaliseRails.project_id)
+        expect(process.status).to eq('queued')
+      end
+
+      it 'handles too many requests' do
+        allow_project_id '672198945b7d72fc048021.15940510'
+        allow(LokaliseRails).to receive(:max_retries_export).and_return(2)
+
+        fake_client = double
+        allow(fake_client).to receive(:upload_file).and_raise(Lokalise::Error::TooManyRequests)
+        allow(fake_class).to receive(:api_client).and_return(fake_client)
+
+        expect(-> { fake_class.export! }).to raise_error(Lokalise::Error::TooManyRequests, /Gave up after 2 retries/i)
+        expect(LokaliseRails).to have_received(:max_retries_export).exactly(3).times
+        expect(fake_class).to have_received(:api_client).exactly(3).times
+        expect(fake_client).to have_received(:upload_file).exactly(3).times
+      end
+    end
+  end
+
   context 'with one translation file' do
     before :all do
       add_translation_files!
@@ -26,6 +65,7 @@ describe LokaliseRails::TaskDefinition::Exporter do
 
         expect(process.project_id).to eq(LokaliseRails.project_id)
         expect(process.status).to eq('queued')
+        expect(LokaliseRails.max_retries_export).to eq(5)
       end
 
       it 'sends a proper API request when a different branch is provided' do

data/spec/lib/lokalise_rails/task_definition/importer_spec.rb

@@ -17,13 +17,14 @@ describe LokaliseRails::TaskDefinition::Importer do
 
   describe '.download_files' do
     it 'returns a proper download URL' do
-      allow_project_id '189934715f57a162257d74.88352370' do
+      allow_project_id '672198945b7d72fc048021.15940510' do
         response = VCR.use_cassette('download_files') do
           described_class.download_files
         end
 
-        expect(response['project_id']).to eq('189934715f57a162257d74.88352370')
+        expect(response['project_id']).to eq('672198945b7d72fc048021.15940510')
         expect(response['bundle_url']).to include('s3-eu-west-1.amazonaws.com')
+        expect(described_class.api_client.enable_compression).to eq(true)
       end
     end
 

data/spec/lib/lokalise_rails_spec.rb

@@ -58,6 +58,11 @@ describe LokaliseRails do
       fake_class.import_safe_mode = true
     end
 
+    it 'is possible to set max_retries_export' do
+      allow(fake_class).to receive(:max_retries_export=).with(10)
+      fake_class.max_retries_export = 10
+    end
+
     it 'is possible to set api_token' do
       allow(fake_class).to receive(:api_token=).with('abc')
       fake_class.api_token = 'abc'

data/spec/support/file_manager.rb

@@ -24,16 +24,20 @@ module FileManager
     locales_dir.count { |file| File.file?(file) }
   end
 
-  def add_translation_files!(with_ru: false)
+  def add_translation_files!(with_ru: false, additional: nil)
     FileUtils.mkdir_p "#{Rails.root}/config/locales/nested"
-    File.open("#{Rails.root}/config/locales/nested/en.yml", 'w+:UTF-8') do |f|
-      f.write en_data
-    end
+    open_and_write('config/locales/nested/en.yml') { |f| f.write en_data }
 
     return unless with_ru
 
-    File.open("#{Rails.root}/config/locales/ru.yml", 'w+:UTF-8') do |f|
-      f.write ru_data
+    open_and_write('config/locales/ru.yml') { |f| f.write ru_data }
+
+    return unless additional
+
+    additional.times do |i|
+      data = {'en' => {"key_#{i}" => "value #{i}"}}
+
+      open_and_write("config/locales/en_#{i}.yml") { |f| f.write data.to_yaml }
     end
   end
 
@@ -46,9 +50,13 @@ module FileManager
       end
     DATA
 
-    File.open("#{Rails.root}/config/lokalise_rails.rb", 'w+:UTF-8') do |f|
-      f.write data
-    end
+    open_and_write('config/lokalise_rails.rb') { |f| f.write data }
+  end
+
+  def open_and_write(rel_path, &block)
+    return unless block
+
+    File.open("#{Rails.root}/#{rel_path}", 'w+:UTF-8', &block)
   end
 
   def remove_config

data/spec/support/spec_addons.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SpecAddons
-  def allow_project_id(value = ENV['LOKALISE_PROJECT_ID'])
+  def allow_project_id(value = '189934715f57a162257d74.88352370')
     allow(LokaliseRails).to receive(:project_id).and_return(value)
     return unless block_given?
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lokalise_rails
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 2.0.0.rc1
 platform: ruby
 authors:
 - Ilya Bodrov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-29 00:00:00.000000000 Z
+date: 2021-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-lokalise-api
@@ -255,11 +255,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.2.24
+rubygems_version: 3.2.25
 signing_key:
 specification_version: 4
 summary: Lokalise integration for Ruby on Rails