lokalise_rails 2.0.0.rc1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8c05601cd8f8684c1523aa4b92d87d10afad700c50d197e40b6886fd4bc9412
-  data.tar.gz: ddc3d0616655b4bda4a40ecf1a96af9c17c5ef54d7c372a05bdf9eb29c2dcdee
+  metadata.gz: 9ac898ae13038db9d3426f367e3cba3b54c41e35797dfb56dd5da5595959e728
+  data.tar.gz: 2bf9f24929b226b8994b3f4648f67b5da59aa0b68687cb7f597f531ac2d7bb99
 SHA512:
-  metadata.gz: 1bfb463dbed7826bf9cad245c53cca76b5ec0c0bc35ddeb42c24c0e9789e93d77adf45e12c772dfcb764e60ffbe3724ecbebb0106bbead653030db1a458df275
-  data.tar.gz: aadd4c0108d6871683c90f4d0d8647f80f072a34e0d4597ec9d4fcb97a9e9494e085b6fe93481925d71a594e0a110ef6575d0499ede0e407d517727bdb0b9baf
+  metadata.gz: 93be5d1f8379908fbf6e3ac7a1f44534373779d8aa3e375f456ef7a83454a07babe3971aa364afe374759e54571d036c60c68e16f14c81397128770573058a9a
+  data.tar.gz: 84c42e122da897f7103a16d11e9a5e066cee28c2b6413c44ec3e459868dfe89d8c5005f3fa644024bea7dfd7c49233de5fecc67746016485f467febe91b3fa52

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 2.0.0 (19-Aug-21)
+
+* Add exponential backoff mechanism for file imports to comply with the upcoming API changes ("rate limiting"), see below for more details.
+* Add `max_retries_import` option with a default value of `5`.
+
 ## 2.0.0.rc1 (12-Aug-21)
 
 * **Lokalise is introducing API rate limiting.** Access to all endpoints will be limited to 6 requests per second from 14 September, 2021. This limit is applied per API token and per IP address. If you exceed the limit, a 429 HTTP status code will be returned and a `Lokalise::Error::TooManyRequests` exception will be raised. Therefore, to overcome this issue LokaliseRails is introducing an exponential backoff mechanism for file exports. Why? Because if you have, say, 10 translation files, we'll have to send 10 separate API requests which will probably mean exceeding the limit (this is not the case with importing as we're always receiving a single archive). Thus, if the HTTP status code 429 was received, we'll do the following:

data/README.md

@@ -113,6 +113,7 @@ Options are specified in the `config/lokalise_rails.rb` file.
 
 Full list of available import options [can be found in the official API documentation](https://app.lokalise.com/api2docs/curl/#transition-download-files-post).
 * `import_safe_mode` (`boolean`) - default to `false`. When this option is enabled, the import task will check whether the directory set with `locales_path` is empty or not. If it is not empty, you will be prompted to continue.
+* `max_retries_import` (`integer`) - this option is introduced to properly handle Lokalise API rate limiting. If the HTTP status code 429 (too many requests) has been received, LokaliseRails will apply an exponential backoff mechanism with a very simple formula: `2 ** retries`. If the maximum number of retries has been reached, a `Lokalise::Error::TooManyRequests` exception will be raised and the export operation will be halted.
 
 ### Export settings
 
@@ -134,7 +135,7 @@ en_US:
 c.skip_file_export = ->(file) { f.split[1].to_s.include?('fr') }
 ```
 
-* `max_retries_export` (`integer`) - this option is introduced to properly handle Lokalise API rate limiting during file exporting. If the HTTP status code 429 (too many requests) has been received, LokaliseRails will apply an exponential backoff mechanism with a very simple formula: `2 ** retries` (initially `retries` is `0`). If the maximum number of retries has been reached, a `Lokalise::Error::TooManyRequests` exception will be raised and the export operation will be halted. By default, LokaliseRails will make up to `5` retries which potentially means `1 + 2 + 4 + 8 + 16 + 32 = 63` seconds of waiting time. If the `max_retries_export` is less than `1`, LokaliseRails will not perform any retries and give up immediately after receiving error 429.
+* `max_retries_export` (`integer`) - this option is introduced to properly handle Lokalise API rate limiting. If the HTTP status code 429 (too many requests) has been received, LokaliseRails will apply an exponential backoff mechanism with a very simple formula: `2 ** retries` (initially `retries` is `0`). If the maximum number of retries has been reached, a `Lokalise::Error::TooManyRequests` exception will be raised and the export operation will be halted. By default, LokaliseRails will make up to `5` retries which potentially means `1 + 2 + 4 + 8 + 16 + 32 = 63` seconds of waiting time. If the `max_retries_export` is less than `1`, LokaliseRails will not perform any retries and give up immediately after receiving error 429.
 
 ### Settings to work with formats other than YAML
 

data/lib/generators/templates/lokalise_rails_config.rb

@@ -19,6 +19,9 @@ LokaliseRails.config do |c|
   # Provide maximum number of retries for file exporting:
   # c.max_retries_export = 5
 
+  # Provide maximum number of retries for file importing:
+  # c.max_retries_import = 5
+
   # Import options have the following defaults:
   # c.import_opts = {
   #   format: 'yaml',

data/lib/lokalise_rails.rb

@@ -10,7 +10,8 @@ module LokaliseRails
     attr_accessor :api_token, :project_id
     attr_writer :import_opts, :import_safe_mode, :export_opts, :locales_path,
                 :file_ext_regexp, :skip_file_export, :branch, :timeouts,
-                :translations_loader, :translations_converter, :lang_iso_inferer, :max_retries_export
+                :translations_loader, :translations_converter, :lang_iso_inferer,
+                :max_retries_export, :max_retries_import
 
     # Main interface to provide configuration options for rake tasks
     def config
@@ -37,6 +38,11 @@ module LokaliseRails
       @max_retries_export || 5
     end
 
+    # Maximum number of retries for file importing
+    def max_retries_import
+      @max_retries_import || 5
+    end
+
     # Regular expression used to select translation files with proper extensions
     def file_ext_regexp
       @file_ext_regexp || /\.ya?ml\z/i

data/lib/lokalise_rails/task_definition/base.rb

@@ -58,6 +58,22 @@ module LokaliseRails
         def project_id_with_branch
           "#{LokaliseRails.project_id}:#{LokaliseRails.branch}"
         end
+
+        # Sends request with exponential backoff mechanism
+        def with_exp_backoff(max_retries)
+          return unless block_given?
+
+          retries = 0
+          begin
+            yield
+          rescue Lokalise::Error::TooManyRequests => e
+            raise(e.class, "Gave up after #{retries} retries") if retries >= max_retries
+
+            sleep 2**retries
+            retries += 1
+            retry
+          end
+        end
       end
     end
   end

data/lib/lokalise_rails/task_definition/exporter.rb

@@ -27,15 +27,8 @@ module LokaliseRails
         # Performs the actual file uploading to Lokalise. If the API rate limit is exceeed,
         # applies exponential backoff
         def do_upload(f_path, r_path)
-          retries = 0
-          begin
+          with_exp_backoff(LokaliseRails.max_retries_export) do
             api_client.upload_file project_id_with_branch, opts(f_path, r_path)
-          rescue Lokalise::Error::TooManyRequests => e
-            raise(e.class, "Gave up after #{retries} retries") if retries >= LokaliseRails.max_retries_export
-
-            sleep 2**retries
-            retries += 1
-            retry
           end
         end
 

data/lib/lokalise_rails/task_definition/importer.rb

@@ -26,13 +26,14 @@ module LokaliseRails
           true
         end
 
-        # Downloads files from Lokalise using the specified options
+        # Downloads files from Lokalise using the specified options.
+        # Utilizes exponential backoff if "too many requests" error is received
         #
         # @return [Hash]
         def download_files
-          opts = LokaliseRails.import_opts
-
-          api_client.download_files project_id_with_branch, opts
+          with_exp_backoff(LokaliseRails.max_retries_import) do
+            api_client.download_files project_id_with_branch, LokaliseRails.import_opts
+          end
         rescue StandardError => e
           raise e.class, "There was an error when trying to download files: #{e.message}"
         end

data/lib/lokalise_rails/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LokaliseRails
-  VERSION = '2.0.0.rc1'
+  VERSION = '2.0.0'
 end

data/spec/lib/lokalise_rails/task_definition/exporter_spec.rb

@@ -39,7 +39,7 @@ describe LokaliseRails::TaskDefinition::Exporter do
         allow(fake_class).to receive(:api_client).and_return(fake_client)
 
         expect(-> { fake_class.export! }).to raise_error(Lokalise::Error::TooManyRequests, /Gave up after 2 retries/i)
-        expect(LokaliseRails).to have_received(:max_retries_export).exactly(3).times
+        expect(LokaliseRails).to have_received(:max_retries_export).exactly(1).times
         expect(fake_class).to have_received(:api_client).exactly(3).times
         expect(fake_client).to have_received(:upload_file).exactly(3).times
       end

data/spec/lib/lokalise_rails/task_definition/importer_spec.rb

@@ -25,6 +25,7 @@ describe LokaliseRails::TaskDefinition::Importer do
         expect(response['project_id']).to eq('672198945b7d72fc048021.15940510')
         expect(response['bundle_url']).to include('s3-eu-west-1.amazonaws.com')
         expect(described_class.api_client.enable_compression).to eq(true)
+        expect(LokaliseRails.max_retries_import).to eq(5)
       end
     end
 
@@ -38,6 +39,22 @@ describe LokaliseRails::TaskDefinition::Importer do
   end
 
   describe '.import!' do
+    let(:fake_class) { described_class }
+
+    it 'handles too many requests' do
+      allow_project_id '672198945b7d72fc048021.15940510'
+      allow(LokaliseRails).to receive(:max_retries_import).and_return(2)
+
+      fake_client = double
+      allow(fake_client).to receive(:download_files).and_raise(Lokalise::Error::TooManyRequests)
+      allow(fake_class).to receive(:api_client).and_return(fake_client)
+
+      expect(-> { fake_class.import! }).to raise_error(Lokalise::Error::TooManyRequests, /Gave up after 2 retries/i)
+      expect(LokaliseRails).to have_received(:max_retries_import).exactly(1).times
+      expect(fake_class).to have_received(:api_client).exactly(3).times
+      expect(fake_client).to have_received(:download_files).exactly(3).times
+    end
+
     it 'halts when the API key is not set' do
       allow(LokaliseRails).to receive(:api_token).and_return(nil)
       expect(-> { described_class.import! }).to raise_error(LokaliseRails::Error, /API token is not set/i)

data/spec/lib/lokalise_rails_spec.rb

@@ -63,6 +63,11 @@ describe LokaliseRails do
       fake_class.max_retries_export = 10
     end
 
+    it 'is possible to set max_retries_import' do
+      allow(fake_class).to receive(:max_retries_import=).with(10)
+      fake_class.max_retries_import = 10
+    end
+
     it 'is possible to set api_token' do
       allow(fake_class).to receive(:api_token=).with('abc')
       fake_class.api_token = 'abc'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lokalise_rails
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc1
+  version: 2.0.0
 platform: ruby
 authors:
 - Ilya Bodrov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-12 00:00:00.000000000 Z
+date: 2021-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-lokalise-api
@@ -255,9 +255,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.2.25
 signing_key: