i18n-tasks 1.0.15 → 1.1.0

data/lib/i18n/tasks/translators/google_translator.rb

@@ -1,29 +1,42 @@
 # frozen_string_literal: true
 
-require 'i18n/tasks/translators/base_translator'
+require "cgi"
+require "net/http"
+require "uri"
+require "json"
+require "i18n/tasks/translators/base_translator"
+
+# https://cloud.google.com/translate/docs/reference/rest/v2/translate
 
 module I18n::Tasks::Translators
   class GoogleTranslator < BaseTranslator
-    NEWLINE_PLACEHOLDER = '<br id=i18n />'
+    API_ENDPOINT = "https://translation.googleapis.com/language/translate/v2"
+    MAX_TEXTS_PER_REQUEST = 128
+    MAX_CHARS_PER_REQUEST = 30_000
+
+    class Error < StandardError; end
+
+    class RateLimitError < Error; end
+
+    class QuotaExceededError < Error; end
+
     def initialize(*)
-      begin
-        require 'easy_translate'
-      rescue LoadError
-        raise ::I18n::Tasks::CommandError, "Add gem 'easy_translate' to your Gemfile to use this command"
-      end
       super
     end
 
     protected
 
     def translate_values(list, **options)
+      html = options[:html].present?
       result = restore_newlines(
-        EasyTranslate.translate(
-          replace_newlines_with_placeholder(list, options[:html]),
-          options,
-          format: options[:html] ? :html : :text
+        translate(
+          texts: replace_newlines_with_placeholder(list),
+          api_key: api_key,
+          to: options[:to],
+          from: options[:from],
+          html:
         ),
-        options[:html]
+        html:
       )
 
       @progress_bar.progress += result.size
@@ -40,15 +53,15 @@ module I18n::Tasks::Translators
     end
 
     def options_for_html
-      { html: true }
+      {html: true}
     end
 
     def options_for_plain
-      { format: 'text' }
+      {format: "text"}
     end
 
     def no_results_error_message
-      I18n.t('i18n_tasks.google_translate.errors.no_results')
+      I18n.t("i18n_tasks.google_translate.errors.no_results")
     end
 
     private
@@ -59,19 +72,17 @@ module I18n::Tasks::Translators
         # fallback with deprecation warning
         if @i18n_tasks.translation_config[:api_key]
           warn_deprecated(
-            'Please rename Google Translate API Key from `api_key` to `google_translate_api_key`.'
+            "Please rename Google Translate API Key from `api_key` to `google_translate_api_key`."
           )
           key ||= translation_config[:api_key]
         end
-        fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.google_translate.errors.no_api_key') if key.blank?
+        fail ::I18n::Tasks::CommandError, I18n.t("i18n_tasks.google_translate.errors.no_api_key") if key.blank?
 
         key
       end
     end
 
-    def replace_newlines_with_placeholder(list, html)
-      return list unless html
-
+    def replace_newlines_with_placeholder(list)
       list.map do |value|
         value.gsub(/\n(\s*)/) do
           "<Z__#{::Regexp.last_match(1)&.length || 0}>"
@@ -79,13 +90,146 @@ module I18n::Tasks::Translators
       end
     end
 
-    def restore_newlines(translations, html)
-      return translations unless html
-
-      translations.map do |translation|
+    def restore_newlines(translations, html:)
+      restored = translations.map do |translation|
         translation.gsub(/<Z__(\d+)>/) do
-          "\n#{' ' * ::Regexp.last_match(1).to_i}"
+          "\n#{" " * ::Regexp.last_match(1).to_i}"
+        end
+      end
+
+      # Need to unescape if translating HTML content
+      html ? restored.map { |t| CGI.unescapeHTML(t) } : restored
+    end
+
+    # @param texts [Array<String>] texts to translate
+    # @param api_key [String] Google Translate API key (required)
+    # @param to [String] target language code (required)
+    # @param format [Symbol] :text or :html (required)
+    # @param from [String] source language code
+    # @return [Array<String>] translated texts
+    def translate(texts:, api_key:, to:, html:, from: nil)
+      texts = Array(texts)
+
+      return [] if texts.empty?
+
+      raise ArgumentError, "api_key is required" if api_key.nil? || api_key.empty?
+
+      raise ArgumentError, "target language (:to) is required" if to.nil? || to.empty?
+
+      # Split into batches if needed
+      batches = batch_texts(texts)
+
+      batches.flat_map do |batch|
+        translate_batch(texts: batch, api_key:, to:, from:, html:)
+      end
+    end
+
+    def batch_texts(texts)
+      batches = []
+      current_batch = []
+      current_chars = 0
+
+      texts.each do |text|
+        text_length = text.to_s.length
+
+        # If adding this text would exceed limits, start new batch
+        if current_batch.size >= MAX_TEXTS_PER_REQUEST ||
+            (current_chars + text_length > MAX_CHARS_PER_REQUEST && current_batch.any?)
+          batches << current_batch
+          current_batch = []
+          current_chars = 0
         end
+
+        current_batch << text
+        current_chars += text_length
+      end
+
+      batches << current_batch if current_batch.any?
+      batches
+    end
+
+    def translate_batch(texts:, api_key:, to:, from:, html:)
+      uri = URI(API_ENDPOINT)
+      # Provide API key via query string as expected by v2 API
+      uri.query = URI.encode_www_form(key: api_key)
+
+      body = {
+        q: texts,
+        target: to,
+        format: html ? "html" : "text"
+      }
+
+      # Add source language if specified
+      if from
+        body[:source] = from
+      end
+
+      response = make_request(uri, body)
+      parse_response(response, texts.size)
+    end
+
+    def make_request(uri, body)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      http.read_timeout = 60
+      http.open_timeout = 30
+
+      request = Net::HTTP::Post.new(uri.request_uri)
+      request["Content-Type"] = "application/json"
+      request["Accept"] = "application/json"
+      request["User-Agent"] = "i18n-tasks/GoogleTranslateApi"
+      request.body = JSON.generate(body)
+
+      response = http.request(request)
+
+      handle_error_response(response) unless response.is_a?(Net::HTTPSuccess)
+      response
+    end
+
+    def parse_response(response, expected_count)
+      data = JSON.parse(response.body)
+
+      unless data["data"] && data["data"]["translations"]
+        raise Error, "Unexpected API response format"
+      end
+
+      translations = data["data"]["translations"].map do |translation|
+        translation["translatedText"]
+      end
+
+      if translations.size != expected_count
+        raise Error, "Expected #{expected_count} translations, got #{translations.size}"
+      end
+
+      translations
+    rescue JSON::ParserError => e
+      raise Error, "Failed to parse API response: #{e.message}"
+    end
+
+    def handle_error_response(response)
+      begin
+        error_data = JSON.parse(response.body)
+        error_message = error_data.dig("error", "message") || "Unknown error"
+        error_code = error_data.dig("error", "code")
+      rescue JSON::ParserError
+        error_message = response.body
+        error_code = response.code
+      end
+
+      status = response.code.to_i
+      diagnostic = "(status=#{status}, code=#{error_code})"
+      case status
+      when 400
+        raise Error, "Bad request #{diagnostic}: #{error_message}"
+      when 401, 403
+        raise Error, "Authentication/authorization failed #{diagnostic}: #{error_message}. " \
+          "Ensure the Cloud Translation API v2 is enabled, billing is active, and the key is unrestricted for this API."
+      when 429
+        raise RateLimitError, "Rate limit exceeded #{diagnostic}: #{error_message}"
+      when 503
+        raise QuotaExceededError, "Quota exceeded #{diagnostic}: #{error_message}"
+      else
+        raise Error, "API error #{diagnostic}: #{error_message}"
       end
     end
   end

data/lib/i18n/tasks/translators/openai_translator.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
-require 'i18n/tasks/translators/base_translator'
-require 'active_support/core_ext/string/filters'
+require "i18n/tasks/translators/base_translator"
+require "active_support/core_ext/string/filters"
 
 module I18n::Tasks::Translators
   class OpenAiTranslator < BaseTranslator
+    include ::I18n::Tasks::Data::LanguageNames
+
     # max allowed texts per request
     BATCH_SIZE = 50
     DEFAULT_SYSTEM_PROMPT = <<~PROMPT.squish
@@ -26,7 +28,7 @@ module I18n::Tasks::Translators
 
     def initialize(*)
       begin
-        require 'openai'
+        require "openai"
       rescue LoadError
         raise ::I18n::Tasks::CommandError, "Add gem 'ruby-openai' to your Gemfile to use this command"
       end
@@ -49,7 +51,7 @@ module I18n::Tasks::Translators
     end
 
     def no_results_error_message
-      I18n.t('i18n_tasks.openai_translate.errors.no_results')
+      I18n.t("i18n_tasks.openai_translate.errors.no_results")
     end
 
     private
@@ -61,29 +63,39 @@ module I18n::Tasks::Translators
     def api_key
       @api_key ||= begin
         key = @i18n_tasks.translation_config[:openai_api_key]
-        fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.openai_translate.errors.no_api_key') if key.blank?
+        fail ::I18n::Tasks::CommandError, I18n.t("i18n_tasks.openai_translate.errors.no_api_key") if key.blank?
 
         key
       end
     end
 
     def model
-      @model ||= @i18n_tasks.translation_config[:openai_model].presence || 'gpt-4o-mini'
+      @model ||= @i18n_tasks.translation_config[:openai_model].presence || "gpt-4o-mini"
+    end
+
+    def temperature
+      @temperature ||= @i18n_tasks.translation_config[:openai_temperature].presence || 0.0
+    end
+
+    def system_prompt(to_locale)
+      prompt = if locale_prompts[to_locale].present?
+        locale_prompts[to_locale]
+      else
+        @i18n_tasks.translation_config[:openai_system_prompt].presence || DEFAULT_SYSTEM_PROMPT
+      end
+
+      prompt.concat("\n#{JSON_FORMAT_INSTRUCTIONS_SYSTEM_PROMPT}")
     end
 
-    def system_prompt
-      @system_prompt ||=
-        (@i18n_tasks.translation_config[:openai_system_prompt].presence || DEFAULT_SYSTEM_PROMPT)
-        .concat("\n#{JSON_FORMAT_INSTRUCTIONS_SYSTEM_PROMPT}")
-      @system_prompt
+    def locale_prompts
+      @locale_prompts ||= @i18n_tasks.translation_config[:openai_locale_prompts] || {}
     end
 
     def translate_values(list, from:, to:)
       results = []
 
       list.each_slice(BATCH_SIZE) do |batch|
-        translations = translate(batch, from, to)
-        result = JSON.parse(translations)
+        result = translate(batch, from, to)
         results << result
 
         @progress_bar.progress += result.size
@@ -97,33 +109,32 @@ module I18n::Tasks::Translators
         parameters: {
           model: model,
           messages: build_messages(values, from, to),
-          temperature: 0.0,
-          response_format: { type: 'json_object' }
+          temperature: temperature,
+          response_format: {type: "json_object"}
         }
       )
 
-      translations = response.dig('choices', 0, 'message', 'content')
-      error = response['error']
+      translations = response.dig("choices", 0, "message", "content")
+      error = response["error"]
 
       fail "AI error: #{error}" if error.present?
 
       # Extract the array from the JSON object response
-      result = JSON.parse(translations)
-      result['translations'].to_json
+      JSON.parse(translations)["translations"]
     end
 
     def build_messages(values, from, to)
       [
         {
-          role: 'system',
-          content: format(system_prompt, from: from, to: to)
+          role: "system",
+          content: format(system_prompt(to), from: language_name(from), to: language_name(to))
         },
         {
-          role: 'user',
+          role: "user",
           content: "Translate this array: \n\n\n"
         },
         {
-          role: 'user',
+          role: "user",
           content: values.to_json
         }
       ]

data/lib/i18n/tasks/translators/watsonx_translator.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'i18n/tasks/translators/base_translator'
-require 'active_support/core_ext/string/filters'
+require "i18n/tasks/translators/base_translator"
+require "active_support/core_ext/string/filters"
 
 module I18n::Tasks::Translators
   class WatsonxTranslator < BaseTranslator
@@ -40,7 +40,7 @@ module I18n::Tasks::Translators
     end
 
     def no_results_error_message
-      I18n.t('i18n_tasks.watsonx_translate.errors.no_results')
+      I18n.t("i18n_tasks.watsonx_translate.errors.no_results")
     end
 
     private
@@ -52,7 +52,7 @@ module I18n::Tasks::Translators
     def api_key
       @api_key ||= begin
         key = @i18n_tasks.translation_config[:watsonx_api_key]
-        fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.watsonx_translate.errors.no_api_key') if key.blank?
+        fail ::I18n::Tasks::CommandError, I18n.t("i18n_tasks.watsonx_translate.errors.no_api_key") if key.blank?
 
         key
       end
@@ -63,7 +63,7 @@ module I18n::Tasks::Translators
         project_id = @i18n_tasks.translation_config[:watsonx_project_id]
         if project_id.blank?
           fail ::I18n::Tasks::CommandError,
-               I18n.t('i18n_tasks.watsonx_translate.errors.no_project_id')
+            I18n.t("i18n_tasks.watsonx_translate.errors.no_project_id")
         end
 
         project_id
@@ -71,7 +71,7 @@ module I18n::Tasks::Translators
     end
 
     def model
-      @model ||= @i18n_tasks.translation_config[:watsonx_model].presence || 'meta-llama/llama-3-2-90b-vision-instruct'
+      @model ||= @i18n_tasks.translation_config[:watsonx_model].presence || "meta-llama/llama-3-2-90b-vision-instruct"
     end
 
     def system_prompt
@@ -94,11 +94,11 @@ module I18n::Tasks::Translators
 
     def translate(values, from, to)
       prompt = [
-        '<|eot_id|><|start_header_id|>system<|end_header_id|>',
+        "<|eot_id|><|start_header_id|>system<|end_header_id|>",
         format(system_prompt, from: from, to: to),
-        '<|eot_id|><|start_header_id|>user<|end_header_id|>Translate this array:',
+        "<|eot_id|><|start_header_id|>user<|end_header_id|>Translate this array:",
         "<|eot_id|><|start_header_id|>user<|end_header_id|>#{values.to_json}",
-        '<|eot_id|><|start_header_id|>assistant<|end_header_id|>'
+        "<|eot_id|><|start_header_id|>assistant<|end_header_id|>"
       ].join
 
       response = translator.generate_text(
@@ -111,18 +111,18 @@ module I18n::Tasks::Translators
           repetition_penalty: 1
         }
       )
-      response.dig('results', 0, 'generated_text')
+      response.dig("results", 0, "generated_text")
     end
   end
 end
 
 class WatsonxClient
-  WATSONX_BASE_URL = 'https://us-south.ml.cloud.ibm.com/ml/'
-  IBM_CLOUD_IAM_URL = 'https://iam.cloud.ibm.com/identity/token'
+  WATSONX_BASE_URL = "https://us-south.ml.cloud.ibm.com/ml/"
+  IBM_CLOUD_IAM_URL = "https://iam.cloud.ibm.com/identity/token"
 
   def initialize(key:)
     begin
-      require 'faraday'
+      require "faraday"
     rescue LoadError
       raise ::I18n::Tasks::CommandError, "Add gem 'faraday' to your Gemfile to use this command"
     end
@@ -137,7 +137,7 @@ class WatsonxClient
   end
 
   def generate_text(**opts)
-    @http.post('v1/text/generation?version=2024-05-20', **opts).body
+    @http.post("v1/text/generation?version=2024-05-20", **opts).body
   end
 
   private
@@ -147,9 +147,9 @@ class WatsonxClient
       conn.use Faraday::Response::RaiseError
       conn.response :json
       conn.params = {
-        grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
+        grant_type: "urn:ibm:params:oauth:grant-type:apikey",
         apikey: key
       }
-    end.post.body['access_token']
+    end.post.body["access_token"]
   end
 end

data/lib/i18n/tasks/translators/yandex_translator.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-require 'i18n/tasks/translators/base_translator'
+require "i18n/tasks/translators/base_translator"
 
 module I18n::Tasks::Translators
   class YandexTranslator < BaseTranslator
     def initialize(*)
       begin
-        require 'yandex-translator'
+        require "yandex-translator"
       rescue LoadError
         raise ::I18n::Tasks::CommandError, "Add gem 'yandex-translator' to your Gemfile to use this command"
       end
@@ -31,24 +31,24 @@ module I18n::Tasks::Translators
     end
 
     def options_for_html
-      { format: 'html' }
+      {format: "html"}
     end
 
     def options_for_plain
-      { format: 'plain' }
+      {format: "plain"}
     end
 
     def no_results_error_message
-      I18n.t('i18n_tasks.yandex_translate.errors.no_results')
+      I18n.t("i18n_tasks.yandex_translate.errors.no_results")
     end
 
     private
 
     # Convert 'es-ES' to 'es'
     def to_yandex_compatible_locale(locale)
-      return locale unless locale.include?('-')
+      return locale unless locale.include?("-")
 
-      locale.split('-', 2).first
+      locale.split("-", 2).first
     end
 
     def translator
@@ -58,7 +58,7 @@ module I18n::Tasks::Translators
     def api_key
       @api_key ||= begin
         key = @i18n_tasks.translation_config[:yandex_api_key]
-        fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.yandex_translate.errors.no_api_key') if key.blank?
+        fail ::I18n::Tasks::CommandError, I18n.t("i18n_tasks.yandex_translate.errors.no_api_key") if key.blank?
 
         key
       end

data/lib/i18n/tasks/unused_keys.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'set'
+require "set"
 
 module I18n
   module Tasks