lokalise_manager 1.2.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33815f129df22f7a27f91689cad944193aca9b992f14e7115a2737088785d3b3
-  data.tar.gz: 1fc5a893bb917bac5afd39f2b35dbbb54286b878dce42f0805135105490520b5
+  metadata.gz: 0f9ca0fe8caf151e84dcc09f29368bc61b5bc3cdb4341ef5c1b21040b764a17a
+  data.tar.gz: 8c9194a836a1261638f2b22d8d2d7553bfbe605325911cb0c0ecef9332e9084c
 SHA512:
-  metadata.gz: 7fecf43ac4f105be85fe2e4548643849b78a239ec523a79b37fec4b6acbeddf319dd4ac594e5caf28e4b174fed76e05b18d3bf6d6343d055daa2115d661b41c4
-  data.tar.gz: 85f6c703a1ee2107cfe165b7c652cc81d2c2f0a0ae2a02af44cfb0e0cf31a185e5cbe3cbe27c8c53d411a8dd4f3046a98b02dadde3d983855e02cead3347cfb4
+  metadata.gz: 7c4b7131ea9d6b60ff4094d0e34c92484818eb3c608a016932426c83bbcab4a06f4c820399f2fa70cc2a78d9a60a73dcf442ec9ba082bdc095ab539b467e1677
+  data.tar.gz: b98c062b4ecc030c37fa1f1ad95126209749c99ec7aba72b616fad380819d55552dc6d4b2a611b94471f79b3748855eefbaab3c9397ab49e9f8e19a6a6afb1ab

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 2.0.0 (27-Jan-22)
+
+* `export!` method is now taking advantage of multi-threading (as Lokalise API allows to send requests in parallel since January 2022)
+* Test with Ruby 3.1.0
+* Other minor fixes
+
 ## 1.2.1 (26-Nov-21)
 
 * Use refinements instead of monkey patching to add hash methods

data/README.md

@@ -1,7 +1,7 @@
 # LokaliseManager
 
 ![Gem](https://img.shields.io/gem/v/lokalise_manager)
-[![Build Status](https://travis-ci.com/bodrovis/lokalise_manager.svg?branch=master)](https://travis-ci.com/github/bodrovis/lokalise_manager)
+![CI](https://github.com/bodrovis/lokalise_manager/actions/workflows/ci.yml/badge.svg)
 [![Test Coverage](https://codecov.io/gh/bodrovis/lokalise_manager/graph/badge.svg)](https://codecov.io/gh/bodrovis/lokalise_manager)
 ![Downloads total](https://img.shields.io/gem/dt/lokalise_manager)
 
@@ -271,4 +271,4 @@ importer.import!
 
 ## License
 
-Copyright (c) [Lokalise team](http://lokalise.com), [Ilya Bodrov](http://bodrovis.tech). License type is [MIT](https://github.com/bodrovis/lokalise_manager/blob/master/LICENSE).
+Copyright (c) [Lokalise team](http://lokalise.com), [Ilya Bodrov](http://bodrovis.tech). License type is [MIT](https://github.com/bodrovis/lokalise_manager/blob/master/LICENSE).

data/lib/lokalise_manager/task_definitions/base.rb

@@ -38,12 +38,13 @@ module LokaliseManager
         client_opts = [config.api_token, {enable_compression: true}.merge(config.timeouts)]
         client_method = config.use_oauth2_token ? :oauth_client : :client
 
-        @api_client ||= ::Lokalise.send(client_method, *client_opts)
+        @api_client = ::Lokalise.send(client_method, *client_opts)
       end
 
       # Resets API client
       def reset_api_client!
-        Lokalise.reset_client!
+        ::Lokalise.reset_client!
+        ::Lokalise.reset_oauth_client!
         @api_client = nil
       end
 

data/lib/lokalise_manager/task_definitions/exporter.rb

@@ -5,6 +5,7 @@ require 'base64'
 module LokaliseManager
   module TaskDefinitions
     class Exporter < Base
+      using LokaliseManager::Utils::ArrayUtils
       # Performs translation file export to Lokalise and returns an array of queued processes
       #
       # @return [Array]
@@ -12,10 +13,13 @@ module LokaliseManager
         check_options_errors!
 
         queued_processes = []
-        each_file do |full_path, relative_path|
-          queued_processes << do_upload(full_path, relative_path)
-        rescue StandardError => e
-          raise e.class, "Error while trying to upload #{full_path}: #{e.message}"
+
+        all_files.in_groups_of(6) do |files_group|
+          parallel_upload(files_group).each do |thr|
+            raise_on_fail thr
+
+            queued_processes.push thr[:process]
+          end
         end
 
         $stdout.print('Task complete!') unless config.silent_mode
@@ -25,18 +29,32 @@ module LokaliseManager
 
       private
 
+      def parallel_upload(files_group)
+        files_group.compact.map do |file_data|
+          do_upload(*file_data)
+        end.map(&:value)
+      end
+
+      def raise_on_fail(thread)
+        raise thread[:error].class, "Error while trying to upload #{thread[:path]}: #{thread[:error].message}" if thread[:status] == :fail
+      end
+
       # Performs the actual file uploading to Lokalise. If the API rate limit is exceeed,
       # applies exponential backoff
       def do_upload(f_path, r_path)
-        with_exp_backoff(config.max_retries_export) do
-          api_client.upload_file project_id_with_branch, opts(f_path, r_path)
+        Thread.new do
+          process = with_exp_backoff(config.max_retries_export) do
+            api_client.upload_file project_id_with_branch, opts(f_path, r_path)
+          end
+          {status: :ok, process: process}
+        rescue StandardError => e
+          {status: :fail, path: f_path, error: e}
         end
       end
 
-      # Processes each translation file in the specified directory
-      def each_file
-        return unless block_given?
-
+      # Gets translation files from the specified directory
+      def all_files
+        files = []
         loc_path = config.locales_path
         Dir["#{loc_path}/**/*"].sort.each do |f|
           full_path = Pathname.new f
@@ -45,8 +63,9 @@ module LokaliseManager
 
           relative_path = full_path.relative_path_from Pathname.new(loc_path)
 
-          yield full_path, relative_path
+          files << [full_path, relative_path]
         end
+        files
       end
 
       # Generates export options

data/lib/lokalise_manager/utils/array_utils.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# Taken from https://github.com/rails/rails/blob/6bfc637659248df5d6719a86d2981b52662d9b50/activesupport/lib/active_support/core_ext/array/grouping.rb
+
+module LokaliseManager
+  module Utils
+    module ArrayUtils
+      refine Array do
+        def in_groups_of(number, fill_with = nil, &block)
+          if number.to_i <= 0
+            raise ArgumentError,
+                  "Group size must be a positive integer, was #{number.inspect}"
+          end
+
+          if fill_with == false
+            collection = self
+          else
+            padding = (number - (size % number)) % number
+            collection = dup.concat(Array.new(padding, fill_with))
+          end
+
+          collection.each_slice(number, &block)
+        end
+      end
+    end
+  end
+end

data/lib/lokalise_manager/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LokaliseManager
-  VERSION = '1.2.1'
+  VERSION = '2.0.0'
 end

data/lib/lokalise_manager.rb

@@ -3,6 +3,7 @@
 require 'yaml'
 
 require 'lokalise_manager/utils/hash_utils'
+require 'lokalise_manager/utils/array_utils'
 
 require 'lokalise_manager/version'
 require 'lokalise_manager/error'

data/lokalise_manager.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.6'
   spec.add_development_dependency 'rubocop', '~> 1.0'
   spec.add_development_dependency 'rubocop-performance', '~> 1.5'
-  spec.add_development_dependency 'rubocop-rspec', '~> 2.6.0'
+  spec.add_development_dependency 'rubocop-rspec', '~> 2.6'
   spec.add_development_dependency 'simplecov', '~> 0.16'
   spec.add_development_dependency 'vcr', '~> 6.0'
   spec.metadata = {

data/spec/lib/lokalise_manager/task_definitions/base_spec.rb

@@ -24,7 +24,7 @@ describe LokaliseManager::TaskDefinitions::Base do
   specify '.reset_client!' do
     expect(described_object.api_client).to be_an_instance_of(Lokalise::Client)
     described_object.reset_api_client!
-    current_client = described_object.instance_variable_get '@api_client'
+    current_client = described_object.instance_variable_get :@api_client
     expect(current_client).to be_nil
   end
 

data/spec/lib/lokalise_manager/task_definitions/exporter_spec.rb

@@ -35,17 +35,19 @@ describe LokaliseManager::TaskDefinitions::Exporter do
       end
 
       it 'handles too many requests' do
+        allow(described_object.config).to receive(:max_retries_export).and_return(1)
         allow(described_object).to receive(:sleep).and_return(0)
 
         fake_client = instance_double('Lokalise::Client')
+        allow(fake_client).to receive(:token).with(any_args).and_return('fake_token')
         allow(fake_client).to receive(:upload_file).with(any_args).and_raise(Lokalise::Error::TooManyRequests)
         allow(described_object).to receive(:api_client).and_return(fake_client)
 
-        expect(-> { described_object.export! }).to raise_error(Lokalise::Error::TooManyRequests, /Gave up after 2 retries/i)
+        expect(-> { described_object.export! }).to raise_error(Lokalise::Error::TooManyRequests, /Gave up after 1 retries/i)
 
-        expect(described_object).to have_received(:sleep).exactly(2).times
-        expect(described_object).to have_received(:api_client).exactly(3).times
-        expect(fake_client).to have_received(:upload_file).exactly(3).times
+        expect(described_object).to have_received(:sleep).exactly(6).times
+        expect(described_object).to have_received(:api_client).at_least(12).times
+        expect(fake_client).to have_received(:upload_file).exactly(12).times
       end
     end
   end
@@ -108,9 +110,9 @@ describe LokaliseManager::TaskDefinitions::Exporter do
       end
     end
 
-    describe '.each_file' do
+    describe '#all_files' do
       it 'yield proper arguments' do
-        expect { |b| described_object.send(:each_file, &b) }.to yield_with_args(
+        expect(described_object.send(:all_files).first).to include(
           Pathname.new(path),
           Pathname.new(relative_name)
         )
@@ -180,30 +182,29 @@ describe LokaliseManager::TaskDefinitions::Exporter do
       end
     end
 
-    describe '.each_file' do
-      it 'yields every translation file' do
-        expect { |b| described_object.send(:each_file, &b) }.to yield_successive_args(
-          [
-            Pathname.new(path),
-            Pathname.new(relative_name)
-          ],
-          [
-            Pathname.new(path_ru),
-            Pathname.new(filename_ru)
-          ]
+    describe '#each_file' do
+      it 'returns all files' do
+        files = described_object.send(:all_files)
+        expect(files[0]).to include(
+          Pathname.new(path),
+          Pathname.new(relative_name)
+        )
+        expect(files[1]).to include(
+          Pathname.new(path_ru),
+          Pathname.new(filename_ru)
         )
       end
 
-      it 'does not yield files that have to be skipped' do
+      it 'does not return files that have to be skipped' do
         allow(described_object.config).to receive(:skip_file_export).twice.and_return(
           ->(f) { f.split[1].to_s.include?('ru') }
         )
-        expect { |b| described_object.send(:each_file, &b) }.to yield_successive_args(
-          [
-            Pathname.new(path),
-            Pathname.new(relative_name)
-          ]
+        files = described_object.send(:all_files)
+        expect(files[0]).to include(
+          Pathname.new(path),
+          Pathname.new(relative_name)
         )
+        expect(files.count).to eq(1)
 
         expect(described_object.config).to have_received(:skip_file_export).twice
       end

data/spec/lib/lokalise_manager/task_definitions/importer_spec.rb

@@ -9,12 +9,16 @@ describe LokaliseManager::TaskDefinitions::Importer do
   let(:loc_path) { described_object.config.locales_path }
   let(:project_id) { ENV['LOKALISE_PROJECT_ID'] }
   let(:local_trans) { "#{Dir.getwd}/spec/fixtures/trans.zip" }
-  let(:faulty_trans) { "#{Dir.getwd}/spec/fixtures/faulty_trans.zip" }
 
-  describe '.open_and_process_zip' do
+  describe '#open_and_process_zip' do
     it 're-raises errors during file processing' do
-      expect(-> { described_object.send(:open_and_process_zip, faulty_trans) }).
-        to raise_error(Psych::DisallowedClass, /Error when trying to process fail\.yml/)
+      entry = double
+      allow(entry).to receive(:name).and_return('fail.yml')
+      allow(described_object).to receive(:data_from).with(entry).and_raise(EncodingError)
+      expect(-> { described_object.send(:process!, entry) }).
+        to raise_error(EncodingError, /Error when trying to process fail\.yml/)
+
+      expect(described_object).to have_received(:data_from)
     end
 
     it 're-raises errors during file opening' do
@@ -23,7 +27,7 @@ describe LokaliseManager::TaskDefinitions::Importer do
     end
   end
 
-  describe '.download_files' do
+  describe '#download_files' do
     it 'returns a proper download URL' do
       response = VCR.use_cassette('download_files') do
         described_object.send :download_files
@@ -111,9 +115,9 @@ describe LokaliseManager::TaskDefinitions::Importer do
 
         expect(result).to be true
 
-        expect(count_translations).to eq(8)
-        expect_file_exist loc_path, 'en.yml'
-        expect_file_exist loc_path, 'ru.yml'
+        expect(count_translations).to eq(24)
+        expect_file_exist loc_path, 'en_1.yml'
+        expect_file_exist loc_path, 'ru_2.yml'
       end
 
       it 'runs import successfully but does not provide any output when silent_mode is enabled' do
@@ -125,8 +129,8 @@ describe LokaliseManager::TaskDefinitions::Importer do
         end
 
         expect(result).to be true
-        expect_file_exist loc_path, 'en.yml'
-        expect_file_exist loc_path, 'ru.yml'
+        expect_file_exist loc_path, 'en_1.yml'
+        expect_file_exist loc_path, 'ru_2.yml'
         expect(described_object.config).to have_received(:silent_mode).at_most(1).times
       end
     end

data/spec/lib/utils/array_utils_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+describe LokaliseManager::Utils::ArrayUtils do
+  using described_class
+  let(:arr) { (1..8).to_a }
+
+  describe '#in_groups_of' do
+    it 'raises an exception when the number is less than 1' do
+      expect(-> { arr.in_groups_of(-1) }).to raise_error(ArgumentError)
+    end
+
+    it 'uses collection itself if fill_with is false' do
+      enum = arr.in_groups_of(5, false)
+      enum.next
+      expect(enum.next.count).to eq(3)
+    end
+  end
+end

data/spec/support/vcr.rb

@@ -6,6 +6,8 @@ VCR.configure do |c|
   c.ignore_hosts 'codeclimate.com'
   c.hook_into :faraday
   c.cassette_library_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'vcr_cassettes')
-  c.filter_sensitive_data('<LOKALISE_TOKEN>') { ENV.fetch('LOKALISE_API_TOKEN') }
+  c.filter_sensitive_data('<LOKALISE_TOKEN>') do |_i|
+    ENV.fetch('LOKALISE_API_TOKEN')
+  end
   c.configure_rspec_metadata!
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lokalise_manager
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Ilya Bodrov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-26 00:00:00.000000000 Z
+date: 2022-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-lokalise-api
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: '2.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -187,6 +187,7 @@ files:
 - lib/lokalise_manager/task_definitions/base.rb
 - lib/lokalise_manager/task_definitions/exporter.rb
 - lib/lokalise_manager/task_definitions/importer.rb
+- lib/lokalise_manager/utils/array_utils.rb
 - lib/lokalise_manager/utils/hash_utils.rb
 - lib/lokalise_manager/version.rb
 - lokalise_manager.gemspec
@@ -195,6 +196,7 @@ files:
 - spec/lib/lokalise_manager/task_definitions/exporter_spec.rb
 - spec/lib/lokalise_manager/task_definitions/importer_spec.rb
 - spec/lib/lokalise_manager_spec.rb
+- spec/lib/utils/array_utils_spec.rb
 - spec/lib/utils/hash_utils_spec.rb
 - spec/spec_helper.rb
 - spec/support/file_manager.rb
@@ -220,7 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.31
+rubygems_version: 3.3.5
 signing_key:
 specification_version: 4
 summary: Lokalise integration for Ruby
@@ -230,6 +232,7 @@ test_files:
 - spec/lib/lokalise_manager/task_definitions/exporter_spec.rb
 - spec/lib/lokalise_manager/task_definitions/importer_spec.rb
 - spec/lib/lokalise_manager_spec.rb
+- spec/lib/utils/array_utils_spec.rb
 - spec/lib/utils/hash_utils_spec.rb
 - spec/spec_helper.rb
 - spec/support/file_manager.rb