web_translate_it 3.2.2 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2344f55bed19e435f8334c66229e58266ff1efe3cf54e9e81f4d17287f42a93d
-  data.tar.gz: 89c6ea08bcc87a43741c8419d5de32e2441f896f12e8338762c15cf84759ab5e
+  metadata.gz: 72c1ce39a90e605ff2b39b63440181d91049014e83d2eba2506185ffe901c3c1
+  data.tar.gz: 5051fbc9a0f55f21351c821098aef8bd4111f1be4d735706389eab5d231c7f60
 SHA512:
-  metadata.gz: c4a9b5b8f2874c7779eedb0d413d826a1d9bccbd8e9db33834ab82a0b8b22b2fb719bf412e7339cb05880018ddc072ee6e70a370824f3fc96f7d6982a2f8396f
-  data.tar.gz: ad5b96143c805efa324ea6119af80dd24517c2e41691b451920934a3deea7115385c13f859ccff7d314c034680f64b95033aba839b1bc4728ff09aa885395a00
+  metadata.gz: 830cf16e7dfdd3ece01f3bce72e5c60d750facaade45aa8626035acf8916b6524590ebd88d0015d7874499177b216c340eb7a0c563665883fb2945ab820d9b97
+  data.tar.gz: b631196b93ffb7a6e4e48b207a958d0e626a6f1e68bcbc825cb5e6db971a727e8cb74405af49f36ad1976275efa2d349b1772ee553a401cbd11df12577e79ff9

data/bin/wti

@@ -42,11 +42,13 @@ when 'pull'
       wti pull [filename] - Pull target language file(s)
       [options] are:
     BANNER
-    opt :locale, 'ISO code of locale(s) to pull, space-separated', type: :string
-    opt :all,    'Pull all files'
-    opt :force,  'Force pull (bypass conditional requests to WTI)'
-    opt :config, 'Path to a configuration file', short: '-c', default: '.wti'
-    opt :debug,  'Display debug information'
+    opt :locale,  'ISO code of locale(s) to pull, space-separated', type: :string
+    opt :all,     'Pull all files'
+    opt :force,   'Force pull (bypass conditional requests to WTI)'
+    opt :threads, 'Number of threads for parallel downloads', type: :integer, default: 10
+    opt :zip,     'Download the project as a zip archive instead of one request per file'
+    opt :config,  'Path to a configuration file', short: '-c', default: '.wti'
+    opt :debug,   'Display debug information'
   end
 when 'push'
   Optimist.options do

data/history.md

@@ -1,3 +1,15 @@
+## Version 3.3.0 / 2026-09-17
+
+* Add `--zip` option to `wti pull`. It downloads the language files through the zip file endpoint: one request per locale instead of one request per file, and a single request when the whole project is pulled. Much faster on projects with many files, for instance when pulling translations in a build. Files already up to date are left alone, only the locales needing an update are requested, and an archive which fails to download fails its own files instead of aborting the pull. #473
+* **`wti pull` now reports download and write failures in its exit code.** `Commands::Base#with_connection` returned the connection rather than the block's value, so `pull` discarded its per-file results and always exited `0`. Builds which were silently green on a partially failed pull will start failing.
+* Support json 3. `multi_json` 1.19 passes parse options positionally, which json 3 rejects, so API error messages came back as raw JSON. Replace `multi_json` with the `json` gem, which ships with Ruby, and run CI against both json 2 and json 3.
+
+## Version 3.2.3 / 2026-04-09
+
+* Handle HTTP 429 (rate limit) errors: add `RateLimitError` class, retry with backoff respecting the `Retry-After` header, and fix garbled error messages when the response body is not JSON.
+* Add `--threads N` option to `wti pull` to control the number of concurrent download threads. Defaults to 10. Use `--threads 1` for sequential pulls.
+* Modernize code for Ruby 3.0+: use `match?` instead of `!~`, `start_with?`/`delete_prefix` instead of index slicing, array difference instead of `reject`, and remove redundant `|| nil` fallbacks.
+
 ## Version 3.2.2 / 2026-03-03
 
 * Replace O(n²) string concatenation loop in `StringUtil#backward_truncate` with `String#ljust`.

data/lib/web_translate_it/api_resource.rb

@@ -9,9 +9,9 @@ module WebTranslateIt
     def initialize(params = {}, connection: nil)
       params = params.transform_keys(&:to_s)
       self.connection   = connection
-      self.id           = params['id'] || nil
-      self.created_at   = params['created_at'] || nil
-      self.updated_at   = params['updated_at'] || nil
+      self.id           = params['id']
+      self.created_at   = params['created_at']
+      self.updated_at   = params['updated_at']
       self.translations = params['translations'] || []
       self.new_record   = true
       assign_attributes(params)
@@ -123,7 +123,7 @@ module WebTranslateIt
     def to_json(*_args, with_translations: false)
       hash = to_hash
       hash['translations'] = translations.map(&:to_hash) if translations.any? && with_translations
-      MultiJson.dump(hash)
+      JSON.generate(hash)
     end
 
     private

data/lib/web_translate_it/auto_fetch.rb

@@ -29,7 +29,7 @@ module WebTranslateIt
     end
 
     def valid_request?(env)
-      env['PATH_INFO'] !~ /\.(js|css|jpeg|jpg|gif|png|woff)$/
+      !env['PATH_INFO'].match?(/\.(js|css|jpeg|jpg|gif|png|woff)$/)
     end
 
   end

data/lib/web_translate_it/commands/base.rb

@@ -29,8 +29,12 @@ module WebTranslateIt
         exit 1
       end
 
-      def with_connection(&block)
-        WebTranslateIt::Connection.new(configuration.api_key, &block)
+      # Yields a connection and returns the block's value. `Connection.new`
+      # returns the connection itself, so the block's value has to be captured.
+      def with_connection
+        result = nil
+        WebTranslateIt::Connection.new(configuration.api_key) { |conn| result = yield conn }
+        result
       end
 
       def run_hook(hook_command, label)

data/lib/web_translate_it/commands/pull.rb

@@ -13,7 +13,9 @@ module WebTranslateIt
         if files.empty?
           puts 'No files to pull.'
         else
-          complete_success = pull_files(files)
+          # `command_options[:zip]`, not `.zip`: Optimist's option hash falls back
+          # to `method_missing`, and `Enumerable#zip` gets there first.
+          complete_success = command_options[:zip] ? pull_archives(files) : pull_files(files)
           run_hook(configuration.after_pull, 'after_pull')
         end
         complete_success
@@ -29,7 +31,7 @@ module WebTranslateIt
 
       def pull_files(files) # rubocop:todo Metrics/AbcSize, Metrics/MethodLength, Naming/PredicateMethod
         time = Time.now
-        results, n_threads = Concurrency.concurrent_batch(files) do |batch|
+        results, n_threads = Concurrency.concurrent_batch(files, max_threads: command_options.threads) do |batch|
           with_connection do |conn|
             batch.map do |file|
               result = file.fetch(conn, command_options.force)
@@ -43,6 +45,78 @@ module WebTranslateIt
         results.all?
       end
 
+      # Pull through the zip file endpoint: one request returns every language
+      # file of the project, which beats one request per file when a project has
+      # many files or many locales.
+      def pull_archives(files)
+        time = Time.now
+        outdated = files.select { |file| file.outdated?(command_options.force) }
+        locales = archive_locales(outdated)
+        saved = save_archives(outdated, locales)
+        results = files.map { |file| saved.fetch(file) { file.skipped } }
+        report_archives(results, locales.size, Time.now - time)
+      end
+
+      def report_archives(results, requests, elapsed) # rubocop:todo Naming/PredicateMethod
+        results.each { |result| print StringUtil.array_to_columns(result.output) }
+        puts "Pulled #{results.size} files in #{elapsed.round(1)}s, using #{requests} archive request(s)."
+        results.all?(&:success)
+      end
+
+      # The zip file endpoint serves either the whole project or a single
+      # locale, so ask for the whole project when every locale is wanted and
+      # fall back to one request per locale otherwise.
+      def archive_locales(files)
+        return [] if files.empty?
+
+        locales = files.map(&:locale).uniq
+        (configuration.target_locales + [configuration.source_locale] - locales).empty? ? [nil] : locales
+      end
+
+      # Returns the result of writing each file, keyed by file.
+      def save_archives(files, locales)
+        return {} if locales.empty?
+
+        by_path = files.to_h { |file| [file.file_path, file] }
+        results, = Concurrency.concurrent_batch(locales, batch_size: 1, max_threads: command_options.threads) do |batch|
+          with_connection { |conn| batch.flat_map { |locale| save_archive(conn, locale, by_path) } }
+        end
+        report_missing(files, results.to_h)
+      end
+
+      # An archive which can't be downloaded or read — an unknown locale, a
+      # server error, a truncated response — fails the files it was carrying
+      # rather than aborting the whole pull and leaving the other threads to be
+      # killed mid-write.
+      def save_archive(connection, locale, by_path)
+        extract(Project.fetch_zip(connection, locale: locale), by_path)
+      rescue StandardError => e
+        by_path.each_value.select { |file| locale.nil? || file.locale == locale }
+               .map { |file| [file, file.failed("An error occured: #{e.message}")] }
+      end
+
+      def extract(archive, by_path)
+        Tempfile.create(['wti-pull', '.zip']) do |tempfile|
+          tempfile.binmode
+          tempfile.write(archive)
+          tempfile.close
+          Zip::File.open(tempfile.path) { |zip| zip.filter_map { |entry| save_entry(entry, by_path) } }
+        end
+      end
+
+      def save_entry(entry, by_path)
+        # rubyzip hands entry names back as binary strings, which never match a
+        # path holding an accented or non-Latin character.
+        file = by_path[entry.name.dup.force_encoding(Encoding::UTF_8)]
+        [file, file.save(entry.get_input_stream(&:read))] if file
+      end
+
+      # A file listed by the project but absent from the archive was deleted
+      # between the two requests. Report it rather than silently skipping it.
+      def report_missing(files, saved)
+        saved.merge(files.difference(saved.keys).to_h { |file| [file, file.save(nil)] })
+      end
+
       def fetch_locales # rubocop:todo Metrics/AbcSize
         locales = if command_options.locale
           warn_unknown_locales(command_options.locale.split)

data/lib/web_translate_it/commands/push.rb

@@ -37,7 +37,7 @@ module WebTranslateIt
         if command_options.locale
           warn_unknown_locales(command_options.locale.split)
         elsif command_options.target
-          configuration.target_locales.reject { |locale| locale == configuration.source_locale }
+          configuration.target_locales - [configuration.source_locale]
         else
           [configuration.source_locale]
         end

data/lib/web_translate_it/project.rb

@@ -21,6 +21,16 @@ module WebTranslateIt
       end
     end
 
+    # Fetch every language file of a project in a single request, as a zip
+    # archive. Pass a locale to narrow the archive down to that locale.
+    def self.fetch_zip(connection, locale: nil)
+      url = "/api/projects/#{connection.api_key}/zip_file"
+      url += "?locale=#{URI.encode_www_form_component(locale)}" if locale
+      Concurrency.with_retries do
+        HttpResponse.handle_response(connection.get(url))
+      end
+    end
+
     def self.create_locale(connection, locale_code)
       Concurrency.with_retries do
         response = connection.post("/api/projects/#{connection.api_key}/locales") { |req| req.set_form_data({'id' => locale_code}, ';') }

data/lib/web_translate_it/string.rb

@@ -16,16 +16,16 @@ module WebTranslateIt
 
     protected
 
-    def assign_attributes(params) # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      self.key          = params['key'] || nil
-      self.plural       = params['plural'] || nil
-      self.type         = params['type'] || nil
-      self.dev_comment  = params['dev_comment'] || nil
-      self.word_count   = params['word_count'] || nil
-      self.status       = params['status'] || nil
-      self.category     = params['category'] || nil
-      self.labels       = params['labels'] || nil
-      self.file         = params['file'] || nil
+    def assign_attributes(params) # rubocop:todo Metrics/AbcSize
+      self.key          = params['key']
+      self.plural       = params['plural']
+      self.type         = params['type']
+      self.dev_comment  = params['dev_comment']
+      self.word_count   = params['word_count']
+      self.status       = params['status']
+      self.category     = params['category']
+      self.labels       = params['labels']
+      self.file         = params['file']
     end
 
     def parse_translation_response(json)

data/lib/web_translate_it/term.rb

@@ -13,8 +13,8 @@ module WebTranslateIt
     protected
 
     def assign_attributes(params)
-      self.text         = params['text'] || nil
-      self.description  = params['description'] || nil
+      self.text         = params['text']
+      self.description  = params['description']
     end
 
     def parse_translation_response(json)

data/lib/web_translate_it/translation_base.rb

@@ -22,7 +22,7 @@ module WebTranslateIt
     end
 
     def to_json(*_args)
-      MultiJson.dump(to_hash)
+      JSON.generate(to_hash)
     end
 
     protected

data/lib/web_translate_it/translation_file.rb

@@ -50,28 +50,44 @@ module WebTranslateIt
     #   file.fetch # returns nothing, with a status 304 Not Modified
     #   file.fetch(true) # force to re-download the file, will return the content of the file with a 200 OK
     #
-    def fetch(connection, force = false) # rubocop:todo Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity
-      display = []
-      if fresh
-        display.push(file_path)
-      else
-        display.push("*#{file_path}")
+    def fetch(connection, force = false)
+      return skipped unless outdated?(force)
+
+      make_directory
+      with_display(display_columns) do
+        response = connection.get(api_url)
+        File.open(file_path, 'wb') { |file| file << response.body } if response.code.to_i == 200
+        response
       end
-      display.push "#{StringUtil.checksumify(local_checksum.to_s)}..#{StringUtil.checksumify(remote_checksum.to_s)}"
-      if !File.exist?(file_path) || force || (remote_checksum != local_checksum)
-
-        dir = File.dirname(file_path)
-        FileUtils.mkpath(dir) unless File.exist?(file_path) || dir == '.'
-        with_display(display) do
-          response = connection.get(api_url)
-          File.open(file_path, 'wb') { |file| file << response.body } if response.code.to_i == 200
-          response
-        end
+    end
 
-      else
-        display.push StringUtil.success('Skipped')
-        Result.new(true, display)
-      end
+    # Write a language file taken out of a project archive, as downloaded by
+    # `wti pull --zip`. `content` is nil when the archive didn't carry the file,
+    # which happens when it was deleted between listing the project and
+    # downloading the archive.
+    def save(content)
+      return failed('Missing from archive') if content.nil?
+
+      make_directory
+      File.open(file_path, 'wb') { |file| file << content }
+      Result.new(true, display_columns.push(StringUtil.success('OK')))
+    rescue StandardError => e
+      failed("An error occured: #{e.message}")
+    end
+
+    # The result of a file which could not be written.
+    def failed(message)
+      Result.new(false, display_columns.push(StringUtil.failure(message)))
+    end
+
+    # Whether the local file is missing or differs from the copy on WebTranslateIt.
+    def outdated?(force = false)
+      !File.exist?(file_path) || force || remote_checksum != local_checksum
+    end
+
+    # The result of leaving an already up-to-date file alone.
+    def skipped
+      Result.new(true, display_columns.push(StringUtil.success('Skipped')))
     end
 
     def fetch_remote_content(connection)
@@ -191,6 +207,18 @@ module WebTranslateIt
 
     private
 
+    # The first two columns of a pull or push line: the file path, prefixed with
+    # a star when its translations are not up to date, and the local and remote
+    # checksums.
+    def display_columns
+      [fresh ? file_path : "*#{file_path}", "#{StringUtil.checksumify(local_checksum.to_s)}..#{StringUtil.checksumify(remote_checksum.to_s)}"]
+    end
+
+    def make_directory
+      dir = File.dirname(file_path)
+      FileUtils.mkpath(dir) unless File.exist?(file_path) || dir == '.'
+    end
+
     def with_display(display)
       Concurrency.with_retries do
         response = yield

data/lib/web_translate_it/util/concurrency.rb

@@ -4,18 +4,27 @@ module WebTranslateIt
 
   module Concurrency
 
-    # Execute a block with automatic retry on Timeout::Error.
+    # Execute a block with automatic retry on Timeout::Error and RateLimitError.
     # Returns the block's return value on success, or re-raises after retries are exhausted.
     def self.with_retries(retries: 3, delay: 5)
       yield
     rescue Timeout::Error
-      puts "Request timeout. Will retry in #{delay} seconds."
-      if (retries -= 1).positive?
-        sleep(delay)
-        retry
-      end
-      raise
+      raise unless (retries -= 1).positive?
+
+      log_retry('Request timeout', delay)
+      retry
+    rescue RateLimitError => e
+      raise unless (retries -= 1).positive?
+
+      log_retry('Rate limited', e.retry_after || delay)
+      retry
+    end
+
+    def self.log_retry(message, wait)
+      puts "#{message}. Will retry in #{wait} seconds."
+      sleep(wait)
     end
+    private_class_method :log_retry
 
     # Process items in parallel using a thread pool.
     # Yields each batch (array of items) to the block; collects return values.

data/lib/web_translate_it/util/http_response.rb

@@ -2,6 +2,17 @@
 
 module WebTranslateIt
 
+  class RateLimitError < StandardError
+
+    attr_reader :retry_after
+
+    def initialize(retry_after: nil)
+      @retry_after = retry_after
+      super("Rate limited#{", retry after #{retry_after}s" if retry_after}")
+    end
+
+  end
+
   module HttpResponse
 
     STATUS_LABELS = {
@@ -31,15 +42,24 @@ module WebTranslateIt
 
     def self.raise_on_error!(response)
       code = response.code.to_i
-      if code >= 400 && code < 500
-        raise "Error: #{MultiJson.load(response.body)['error']}"
-      elsif code == 500
-        raise 'Error: Server temporarily unavailable (Error 500).'
-      elsif code == 503
-        raise 'Error: Locked (another import in progress)'
-      end
+      raise_on_rate_limit!(response) if code == 429
+      raise "Error: #{error_message(response)}" if code >= 400 && code < 500
+      raise 'Error: Server temporarily unavailable (Error 500).' if code == 500
+      raise 'Error: Locked (another import in progress)' if code == 503
     end
-    private_class_method :raise_on_error!
+
+    def self.raise_on_rate_limit!(response)
+      retry_after = response['Retry-After']&.to_i
+      raise RateLimitError.new(retry_after: retry_after)
+    end
+
+    def self.error_message(response)
+      JSON.parse(response.body)['error']
+    rescue StandardError
+      response.body.to_s
+    end
+
+    private_class_method :raise_on_error!, :raise_on_rate_limit!, :error_message
 
   end
 

data/lib/web_translate_it/util/string_util.rb

@@ -25,8 +25,8 @@ class StringUtil
   end
 
   def self.array_to_columns(array)
-    if array[0][0] == '*'
-      "*#{backward_truncate(array[0][1..])} | #{array[1]}  #{array[2]}\n"
+    if array[0].start_with?('*')
+      "*#{backward_truncate(array[0].delete_prefix('*'))} | #{array[1]}  #{array[2]}\n"
     else
       " #{backward_truncate(array[0])} | #{array[1]}  #{array[2]}\n"
     end

data/lib/web_translate_it.rb

@@ -8,9 +8,10 @@ require 'net/http'
 require 'net/https'
 require 'openssl'
 require 'uri'
-require 'multi_json'
 require 'digest/sha1'
 require 'English'
+require 'tempfile'
+require 'zip'
 
 require 'web_translate_it/connection'
 require 'web_translate_it/util'

data/man/wti.1.ron

@@ -45,7 +45,17 @@ These options can be used to change this behaviour:
     The Web Translate It API use HTTP caching to be efficient, and check if your file needs
     to be updated by checking its modification date against the project’s latest activity.
     Use this option to bypass this check.
-    
+
+  * `--zip`:
+    Pull the language files as zip archives, one request per locale instead of one
+    request per file. Pulling the whole project, with `--all` and no ignored or needed
+    locales configured, takes a single request. Much faster on projects with many files.
+    Files already up to date are left alone, and only the locales needing an update
+    are requested.
+
+  * `--threads`:
+    Number of parallel downloads. Defaults to 10.
+
 You may additionally ask for help:
  
   * `-h`, `--help`:

data/readme.md

@@ -177,6 +177,10 @@ Append `--help` for each command for more information. For instance:
     <td>wti pull --force</td>
     <td>Force pull (to bypass WebTranslateIt’s HTTP caching)</td>
   </tr>
+  <tr>
+    <td>wti pull --zip</td>
+    <td>Download the language files as zip archives: one request per locale instead of one per file, or a single request when the whole project is pulled. Much faster on projects with many files</td>
+  </tr>
   <tr>
     <td>wti addlocale fr</td>
     <td>Add a new locale to the project</td>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: web_translate_it
 version: !ruby/object:Gem::Version
-  version: 3.2.2
+  version: 3.3.0
 platform: ruby
 authors:
 - Edouard Briere
@@ -10,19 +10,25 @@ cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: multi_json
+  name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: optimist
   requirement: !ruby/object:Gem::Requirement
@@ -37,6 +43,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
 description: A Command Line Interface tool to push and pull language files to WebTranslateIt.com.
 email: support@webtranslateit.com
 executables:
@@ -113,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.20
 specification_version: 4
 summary: A CLI tool to sync locale files with WebTranslateIt.com.
 test_files: []