lokalise_manager 1.0.0

data/lib/lokalise_manager/task_definitions/exporter.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module LokaliseManager
+  module TaskDefinitions
+    class Exporter < Base
+      # Performs translation file export to Lokalise and returns an array of queued processes
+      #
+      # @return [Array]
+      def export!
+        check_options_errors!
+
+        queued_processes = []
+        each_file do |full_path, relative_path|
+          queued_processes << do_upload(full_path, relative_path)
+        rescue StandardError => e
+          raise e.class, "Error while trying to upload #{full_path}: #{e.message}"
+        end
+
+        $stdout.print 'Task complete!'
+
+        queued_processes
+      end
+
+      private
+
+      # Performs the actual file uploading to Lokalise. If the API rate limit is exceeed,
+      # applies exponential backoff
+      def do_upload(f_path, r_path)
+        with_exp_backoff(config.max_retries_export) do
+          api_client.upload_file project_id_with_branch, opts(f_path, r_path)
+        end
+      end
+
+      # Processes each translation file in the specified directory
+      def each_file
+        return unless block_given?
+
+        loc_path = config.locales_path
+        Dir["#{loc_path}/**/*"].sort.each do |f|
+          full_path = Pathname.new f
+
+          next unless file_matches_criteria? full_path
+
+          relative_path = full_path.relative_path_from Pathname.new(loc_path)
+
+          yield full_path, relative_path
+        end
+      end
+
+      # Generates export options
+      #
+      # @return [Hash]
+      # @param full_p [Pathname]
+      # @param relative_p [Pathname]
+      def opts(full_p, relative_p)
+        content = File.read full_p
+
+        initial_opts = {
+          data: Base64.strict_encode64(content.strip),
+          filename: relative_p,
+          lang_iso: config.lang_iso_inferer.call(content)
+        }
+
+        initial_opts.merge config.export_opts
+      end
+
+      # Checks whether the specified file has to be processed or not
+      #
+      # @return [Boolean]
+      # @param full_path [Pathname]
+      def file_matches_criteria?(full_path)
+        full_path.file? && proper_ext?(full_path) &&
+          !config.skip_file_export.call(full_path)
+      end
+    end
+  end
+end

data/lib/lokalise_manager/task_definitions/importer.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'zip'
+require 'open-uri'
+require 'fileutils'
+
+module LokaliseManager
+  module TaskDefinitions
+    class Importer < Base
+      # Performs translation files import from Lokalise
+      #
+      # @return [Boolean]
+      def import!
+        check_options_errors!
+
+        unless proceed_when_safe_mode?
+          $stdout.print 'Task cancelled!'
+          return false
+        end
+
+        open_and_process_zip download_files['bundle_url']
+
+        $stdout.print 'Task complete!'
+        true
+      end
+
+      private
+
+      # Downloads files from Lokalise using the specified config.
+      # Utilizes exponential backoff if "too many requests" error is received
+      #
+      # @return [Hash]
+      def download_files
+        with_exp_backoff(config.max_retries_import) do
+          api_client.download_files project_id_with_branch, config.import_opts
+        end
+      rescue StandardError => e
+        raise e.class, "There was an error when trying to download files: #{e.message}"
+      end
+
+      # Opens ZIP archive (local or remote) with translations and processes its entries
+      #
+      # @param path [String]
+      def open_and_process_zip(path)
+        Zip::File.open_buffer(open_file_or_remote(path)) do |zip|
+          fetch_zip_entries(zip) { |entry| process!(entry) }
+        end
+      rescue StandardError => e
+        raise e.class, "There was an error when trying to process the downloaded files: #{e.message}"
+      end
+
+      # Iterates over ZIP entries. Each entry may be a file or folder.
+      def fetch_zip_entries(zip)
+        return unless block_given?
+
+        zip.each do |entry|
+          next unless proper_ext? entry.name
+
+          yield entry
+        end
+      end
+
+      # Processes ZIP entry by reading its contents and creating the corresponding translation file
+      def process!(zip_entry)
+        data = data_from zip_entry
+        subdir, filename = subdir_and_filename_for zip_entry.name
+        full_path = "#{config.locales_path}/#{subdir}"
+        FileUtils.mkdir_p full_path
+
+        File.open(File.join(full_path, filename), 'w+:UTF-8') do |f|
+          f.write config.translations_converter.call(data)
+        end
+      rescue StandardError => e
+        raise e.class, "Error when trying to process #{zip_entry&.name}: #{e.message}"
+      end
+
+      # Checks whether the user wishes to proceed when safe mode is enabled and the target directory is not empty
+      #
+      # @return [Boolean]
+      def proceed_when_safe_mode?
+        return true unless config.import_safe_mode && !Dir.empty?(config.locales_path.to_s)
+
+        $stdout.puts "The target directory #{config.locales_path} is not empty!"
+        $stdout.print 'Enter Y to continue: '
+        answer = $stdin.gets
+        answer.to_s.strip == 'Y'
+      end
+
+      # Opens a local file or a remote URL using the provided patf
+      #
+      # @return [String]
+      def open_file_or_remote(path)
+        parsed_path = URI.parse(path)
+        if parsed_path&.scheme&.include?('http')
+          parsed_path.open
+        else
+          File.open path
+        end
+      end
+
+      def data_from(zip_entry)
+        config.translations_loader.call zip_entry.get_input_stream.read
+      end
+    end
+  end
+end

data/lib/lokalise_manager/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module LokaliseManager
+  VERSION = '1.0.0'
+end

data/lib/lokalise_manager.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+require 'lokalise_manager/version'
+require 'lokalise_manager/error'
+require 'lokalise_manager/global_config'
+require 'lokalise_manager/task_definitions/base'
+require 'lokalise_manager/task_definitions/importer'
+require 'lokalise_manager/task_definitions/exporter'
+
+module LokaliseManager
+  class << self
+    # Initializes a new importer client which is used to download
+    # translation files from Lokalise to the current project
+    #
+    # @return [LokaliseManager::TaskDefinitions::Importer]
+    # @param custom_opts [Hash]
+    # @param global_config [Object]
+    def importer(custom_opts = {}, global_config = LokaliseManager::GlobalConfig)
+      LokaliseManager::TaskDefinitions::Importer.new custom_opts, global_config
+    end
+
+    # Initializes a new exporter client which is used to upload
+    # translation files from the current project to Lokalise
+    #
+    # @return [LokaliseManager::TaskDefinitions::Exporter]
+    # @param custom_opts [Hash]
+    # @param global_config [Object]
+    def exporter(custom_opts = {}, global_config = LokaliseManager::GlobalConfig)
+      LokaliseManager::TaskDefinitions::Exporter.new custom_opts, global_config
+    end
+  end
+end

data/lokalise_manager.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require File.expand_path('lib/lokalise_manager/version', __dir__)
+
+Gem::Specification.new do |spec|
+  spec.name                  = 'lokalise_manager'
+  spec.version               = LokaliseManager::VERSION
+  spec.authors               = ['Ilya Bodrov']
+  spec.email                 = ['golosizpru@gmail.com']
+  spec.summary               = 'Lokalise integration for Ruby'
+  spec.description           = 'This gem contains a collection of some common tasks for Lokalise. Specifically, it allows to import/export translation files from/to Lokalise TMS.'
+  spec.homepage              = 'https://github.com/bodrovis/lokalise_manager'
+  spec.license               = 'MIT'
+  spec.platform              = Gem::Platform::RUBY
+  spec.required_ruby_version = '>= 2.5.0'
+
+  spec.files = Dir['README.md', 'LICENSE',
+                   'CHANGELOG.md', 'lib/**/*.rb',
+                   'lib/**/*.rake',
+                   'lokalise_manager.gemspec', '.github/*.md',
+                   'Gemfile', 'Rakefile']
+  spec.test_files       = Dir['spec/**/*.rb']
+  spec.extra_rdoc_files = ['README.md']
+  spec.require_paths    = ['lib']
+
+  spec.add_dependency 'ruby-lokalise-api', '~> 4.0'
+  spec.add_dependency 'rubyzip', '~> 2.3'
+
+  spec.add_development_dependency 'codecov', '~> 0.2'
+  spec.add_development_dependency 'dotenv', '~> 2.5'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rspec', '~> 3.6'
+  spec.add_development_dependency 'rubocop', '~> 1.0'
+  spec.add_development_dependency 'rubocop-performance', '~> 1.5'
+  spec.add_development_dependency 'rubocop-rspec', '~> 2.5.0'
+  spec.add_development_dependency 'simplecov', '~> 0.16'
+  spec.add_development_dependency 'vcr', '~> 6.0'
+end

data/spec/lib/lokalise_manager/global_config_spec.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+describe LokaliseManager::GlobalConfig do
+  let(:fake_class) { class_double(described_class) }
+
+  it 'is possible to provide config' do
+    described_class.config do |c|
+      expect(c).to eq(described_class)
+    end
+  end
+
+  it 'is possible to set project_id' do
+    allow(fake_class).to receive(:project_id=).with('123.abc')
+    fake_class.project_id = '123.abc'
+  end
+
+  it 'is possible to set file_ext_regexp' do
+    allow(fake_class).to receive(:file_ext_regexp=).with(Regexp.new('.*'))
+    fake_class.file_ext_regexp = Regexp.new('.*')
+  end
+
+  it 'is possible to set import_opts' do
+    allow(fake_class).to receive(:import_opts=).with(duck_type(:each))
+    fake_class.import_opts = {
+      format: 'json',
+      indentation: '8sp'
+    }
+  end
+
+  it 'is possible to set export_opts' do
+    allow(fake_class).to receive(:export_opts=).with(duck_type(:each))
+    fake_class.export_opts = {
+      convert_placeholders: true,
+      detect_icu_plurals: true
+    }
+  end
+
+  it 'is possible to set branch' do
+    allow(fake_class).to receive(:branch=).with('custom')
+    fake_class.branch = 'custom'
+  end
+
+  it 'is possible to set timeouts' do
+    allow(fake_class).to receive(:timeouts=).with(duck_type(:each))
+    fake_class.timeouts = {
+      open_timeout: 100,
+      timeout: 500
+    }
+  end
+
+  it 'is possible to set import_safe_mode' do
+    allow(fake_class).to receive(:import_safe_mode=).with(true)
+    fake_class.import_safe_mode = true
+  end
+
+  it 'is possible to set max_retries_export' do
+    allow(fake_class).to receive(:max_retries_export=).with(10)
+    fake_class.max_retries_export = 10
+  end
+
+  it 'is possible to set max_retries_import' do
+    allow(fake_class).to receive(:max_retries_import=).with(10)
+    fake_class.max_retries_import = 10
+  end
+
+  it 'is possible to set api_token' do
+    allow(fake_class).to receive(:api_token=).with('abc')
+    fake_class.api_token = 'abc'
+  end
+
+  it 'is possible to override locales_path' do
+    allow(fake_class).to receive(:locales_path=).with('/demo/path')
+    fake_class.locales_path = '/demo/path'
+  end
+
+  it 'is possible to set skip_file_export' do
+    cond = ->(f) { f.nil? }
+    allow(fake_class).to receive(:skip_file_export=).with(cond)
+    fake_class.skip_file_export = cond
+  end
+
+  it 'is possible to set translations_loader' do
+    runner = ->(f) { f.to_json }
+    allow(fake_class).to receive(:translations_loader=).with(runner)
+    fake_class.translations_loader = runner
+  end
+
+  it 'is possible to set translations_converter' do
+    runner = ->(f) { f.to_json }
+    allow(fake_class).to receive(:translations_converter=).with(runner)
+    fake_class.translations_converter = runner
+  end
+
+  it 'is possible to set lang_iso_inferer' do
+    runner = ->(f) { f.to_json }
+    allow(fake_class).to receive(:lang_iso_inferer=).with(runner)
+    fake_class.lang_iso_inferer = runner
+  end
+end

data/spec/lib/lokalise_manager/task_definitions/base_spec.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+describe LokaliseManager::TaskDefinitions::Base do
+  let(:described_object) { described_class.new }
+
+  describe '.new' do
+    it 'allows to override config' do
+      obj = described_class.new token: 'fake'
+      expect(obj.config.token).to eq('fake')
+    end
+  end
+
+  describe '#config' do
+    it 'allows to update config after initialization' do
+      obj = described_class.new token: 'fake', project_id: '123'
+
+      obj.config.project_id = '345'
+
+      expect(obj.config.project_id).to eq('345')
+      expect(obj.config.token).to eq('fake')
+    end
+  end
+
+  specify '.reset_client!' do
+    expect(described_object.api_client).to be_an_instance_of(Lokalise::Client)
+    described_object.reset_api_client!
+    current_client = described_object.instance_variable_get '@api_client'
+    expect(current_client).to be_nil
+  end
+
+  specify '.project_id_with_branch!' do
+    result = described_object.send :project_id_with_branch
+    expect(result).to be_an_instance_of(String)
+    expect(result).not_to include(':master')
+
+    described_object.config.branch = 'develop'
+    result = described_object.send :project_id_with_branch
+    expect(result).to be_an_instance_of(String)
+    expect(result).to include(':develop')
+  end
+
+  describe '.subdir_and_filename_for' do
+    it 'works properly for longer paths' do
+      path = 'my_path/is/here/file.yml'
+      result = described_object.send(:subdir_and_filename_for, path)
+      expect(result.length).to eq(2)
+      expect(result[0]).to be_an_instance_of(Pathname)
+      expect(result[0].to_s).to eq('my_path/is/here')
+      expect(result[1].to_s).to eq('file.yml')
+    end
+
+    it 'works properly for shorter paths' do
+      path = 'file.yml'
+      result = described_object.send(:subdir_and_filename_for, path)
+      expect(result.length).to eq(2)
+      expect(result[1]).to be_an_instance_of(Pathname)
+      expect(result[0].to_s).to eq('.')
+      expect(result[1].to_s).to eq('file.yml')
+    end
+  end
+
+  describe '.check_options_errors!' do
+    it 'raises an error when the API key is not set' do
+      allow(LokaliseManager::GlobalConfig).to receive(:api_token).and_return(nil)
+
+      expect(-> { described_object.send(:check_options_errors!) }).to raise_error(LokaliseManager::Error, /API token is not set/i)
+
+      expect(LokaliseManager::GlobalConfig).to have_received(:api_token)
+    end
+
+    it 'returns an error when the project_id is not set' do
+      allow_project_id described_object, nil do
+        expect(-> { described_object.send(:check_options_errors!) }).to raise_error(LokaliseManager::Error, /ID is not set/i)
+      end
+    end
+  end
+
+  describe '.proper_ext?' do
+    it 'works properly with path represented as a string' do
+      path = 'my_path/here/file.yml'
+      expect(described_object.send(:proper_ext?, path)).to be true
+    end
+
+    it 'works properly with path represented as a pathname' do
+      path = Pathname.new 'my_path/here/file.json'
+      expect(described_object.send(:proper_ext?, path)).to be false
+    end
+  end
+
+  describe '.api_client' do
+    it 'is possible to set timeouts' do
+      allow(described_object.config).to receive(:timeouts).and_return({
+                                                                        open_timeout: 100,
+                                                                        timeout: 500
+                                                                      })
+
+      expect(described_object.api_client).to be_an_instance_of(Lokalise::Client)
+      expect(described_object.api_client.open_timeout).to eq(100)
+      expect(described_object.api_client.timeout).to eq(500)
+    end
+  end
+end

data/spec/lib/lokalise_manager/task_definitions/exporter_spec.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+describe LokaliseManager::TaskDefinitions::Exporter do
+  let(:filename) { 'en.yml' }
+  let(:path) { "#{Dir.getwd}/locales/nested/#{filename}" }
+  let(:relative_name) { "nested/#{filename}" }
+  let(:project_id) { ENV['LOKALISE_PROJECT_ID'] }
+  let(:described_object) do
+    described_class.new project_id: project_id,
+                        api_token: ENV['LOKALISE_API_TOKEN'],
+                        max_retries_export: 2
+  end
+
+  context 'with many translation files' do
+    before :all do
+      add_translation_files! with_ru: true, additional: 5
+    end
+
+    after :all do
+      rm_translation_files
+    end
+
+    describe '.export!' do
+      it 'sends a proper API request and handles rate limiting' do
+        process = VCR.use_cassette('upload_files_multiple') do
+          described_object.export!
+        end.first
+
+        expect(process.project_id).to eq(project_id)
+        expect(process.status).to eq('queued')
+      end
+
+      it 'handles too many requests' do
+        allow(described_object).to receive(:sleep).and_return(0)
+
+        fake_client = instance_double('Lokalise::Client')
+        allow(fake_client).to receive(:upload_file).with(any_args).and_raise(Lokalise::Error::TooManyRequests)
+        allow(described_object).to receive(:api_client).and_return(fake_client)
+
+        expect(-> { described_object.export! }).to raise_error(Lokalise::Error::TooManyRequests, /Gave up after 2 retries/i)
+
+        expect(described_object).to have_received(:sleep).exactly(2).times
+        expect(described_object).to have_received(:api_client).exactly(3).times
+        expect(fake_client).to have_received(:upload_file).exactly(3).times
+      end
+    end
+  end
+
+  context 'with one translation file' do
+    before :all do
+      add_translation_files!
+    end
+
+    after :all do
+      rm_translation_files
+    end
+
+    describe '.export!' do
+      it 'sends a proper API request' do
+        process = VCR.use_cassette('upload_files') do
+          described_object.export!
+        end.first
+
+        expect(process.project_id).to eq(project_id)
+        expect(process.status).to eq('queued')
+      end
+
+      it 'sends a proper API request when a different branch is provided' do
+        allow(described_object.config).to receive(:branch).and_return('develop')
+
+        process = VCR.use_cassette('upload_files_branch') do
+          described_object.export!
+        end.first
+
+        expect(described_object.config).to have_received(:branch).at_most(2).times
+        expect(process.project_id).to eq(project_id)
+        expect(process.status).to eq('queued')
+      end
+
+      it 'halts when the API key is not set' do
+        allow(described_object.config).to receive(:api_token).and_return(nil)
+
+        expect(-> { described_object.export! }).to raise_error(LokaliseManager::Error, /API token is not set/i)
+        expect(described_object.config).to have_received(:api_token)
+      end
+
+      it 'halts when the project_id is not set' do
+        allow_project_id described_object, nil do
+          expect(-> { described_object.export! }).to raise_error(LokaliseManager::Error, /ID is not set/i)
+        end
+      end
+    end
+
+    describe '.each_file' do
+      it 'yield proper arguments' do
+        expect { |b| described_object.send(:each_file, &b) }.to yield_with_args(
+          Pathname.new(path),
+          Pathname.new(relative_name)
+        )
+      end
+    end
+
+    describe '.opts' do
+      let(:base64content) { Base64.strict_encode64(File.read(path).strip) }
+
+      it 'generates proper options' do
+        resulting_opts = described_object.send(:opts, path, relative_name)
+
+        expect(resulting_opts[:data]).to eq(base64content)
+        expect(resulting_opts[:filename]).to eq(relative_name)
+        expect(resulting_opts[:lang_iso]).to eq('en')
+      end
+
+      it 'allows to redefine options' do
+        allow(described_object.config).to receive(:export_opts).and_return({
+                                                                             detect_icu_plurals: true,
+                                                                             convert_placeholders: true
+                                                                           })
+
+        resulting_opts = described_object.send(:opts, path, relative_name)
+
+        expect(described_object.config).to have_received(:export_opts)
+        expect(resulting_opts[:data]).to eq(base64content)
+        expect(resulting_opts[:filename]).to eq(relative_name)
+        expect(resulting_opts[:lang_iso]).to eq('en')
+        expect(resulting_opts[:detect_icu_plurals]).to be true
+        expect(resulting_opts[:convert_placeholders]).to be true
+      end
+    end
+  end
+
+  context 'with two translation files' do
+    let(:filename_ru) { 'ru.yml' }
+    let(:path_ru) { "#{Dir.getwd}/locales/#{filename_ru}" }
+
+    before :all do
+      add_translation_files! with_ru: true
+    end
+
+    after :all do
+      rm_translation_files
+    end
+
+    describe '.export!' do
+      it 're-raises export errors' do
+        allow_project_id described_object, '542886116159f798720dc4.94769464'
+
+        VCR.use_cassette('upload_files_error') do
+          expect { described_object.export! }.to raise_error(Lokalise::Error::BadRequest, /Unknown `lang_iso`/)
+        end
+      end
+    end
+
+    describe '.opts' do
+      let(:base64content_ru) { Base64.strict_encode64(File.read(path_ru).strip) }
+
+      it 'generates proper options' do
+        resulting_opts = described_object.send(:opts, path_ru, filename_ru)
+
+        expect(resulting_opts[:data]).to eq(base64content_ru)
+        expect(resulting_opts[:filename]).to eq(filename_ru)
+        expect(resulting_opts[:lang_iso]).to eq('ru_RU')
+      end
+    end
+
+    describe '.each_file' do
+      it 'yields every translation file' do
+        expect { |b| described_object.send(:each_file, &b) }.to yield_successive_args(
+          [
+            Pathname.new(path),
+            Pathname.new(relative_name)
+          ],
+          [
+            Pathname.new(path_ru),
+            Pathname.new(filename_ru)
+          ]
+        )
+      end
+
+      it 'does not yield files that have to be skipped' do
+        allow(described_object.config).to receive(:skip_file_export).twice.and_return(
+          ->(f) { f.split[1].to_s.include?('ru') }
+        )
+        expect { |b| described_object.send(:each_file, &b) }.to yield_successive_args(
+          [
+            Pathname.new(path),
+            Pathname.new(relative_name)
+          ]
+        )
+
+        expect(described_object.config).to have_received(:skip_file_export).twice
+      end
+    end
+  end
+end