immosquare-translate 0.1.14 → 0.1.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87dc92e836e2e404cccfa85c01fac2b8edf396aecc1a8a74558f66532942d6fd
-  data.tar.gz: 449976467c208d55e4dc7bf95c1089fae2889a954d64c385186789aabb1bd228
+  metadata.gz: 702066291942b4302f39c2d9a079b8ab0f959fa6aa2236c850e1ceb8f088425e
+  data.tar.gz: b58247526fa01fa6ec747ac808217c1f62db7497b8dd14202cfcf685615094cf
 SHA512:
-  metadata.gz: c89194b90aa0b774308113059769a2d3021a13d7359df80baeab1539562acbd0dc1e8f5088f48d5bb7d66efcf5a4214cf49aa71cbfa609da6013dd080d8bf2be
-  data.tar.gz: 2eb2ee90027ebbbfe1fb38aee306c070b785de2aed44001290225d67f83b25ad872efd09d2910832e68bff65e91507b200ea9bfa865fd8cbbc67bc85a9514c89
+  metadata.gz: 5a29bf48a145db0dafbfd49654377b67b73359b085f9e31d25ef3f0e566e2c9448f0c402734cdfe3c55484fe16a29de4961b4487507b4336a844d1184e7f891a
+  data.tar.gz: 337435d74a74673370ec1c7066a95a11c217c448fd59464a3cfb236b0c65f8bbfc57906557e8a000d9323acd578dc7cb1776fd62f080adde29b3defc121e00da

data/lib/immosquare-translate/shared_methods.rb

@@ -11,12 +11,12 @@ module ImmosquareTranslate
     ##============================================================##
     OPEN_AI_MODELS = [
       {:nickname => "gpt-3.5",      :name => "gpt-3.5-turbo-0125",     :default => false, :window_tokens => 16_385,    :output_tokens => 4_096,  :input_price_for_1m => 0.50,   :output_price_for_1m => 1.50,  :group_size => 75},
-      {:nickname => "gpt-4",        :name => "gpt-4-turbo-2024-04-09", :default => false, :window_tokens => 128_000,   :output_tokens => 4_096,  :input_price_for_1m => 10.00,  :output_price_for_1m => 30.00, :group_size => 75},
-      {:nickname => "gpt-4o-mini",  :name => "gpt-4o-mini",            :default => false, :window_tokens => 128_000,   :output_tokens => 16_384, :input_price_for_1m => 0.15,   :output_price_for_1m => 0.60,  :group_size => 75},
-      {:nickname => "gpt-4o",       :name => "gpt-4o-2024-08-06",      :default => false, :window_tokens => 128_000,   :output_tokens => 16_384, :input_price_for_1m => 2.50,   :output_price_for_1m => 10.00, :group_size => 75},
-      {:nickname => "gpt-4.1-nano", :name => "gpt-4.1-nano",           :default => false, :window_tokens => 1_000_000, :output_tokens => 32_768, :input_price_for_1m => 0.10,   :output_price_for_1m => 0.40,  :group_size => 75},
-      {:nickname => "gpt-4.1-mini", :name => "gpt-4.1-mini",           :default => false, :window_tokens => 1_000_000, :output_tokens => 32_768, :input_price_for_1m => 0.40,   :output_price_for_1m => 1.60,  :group_size => 75},
-      {:nickname => "gpt-4.1",      :name => "gpt-4.1-2025-04-14",     :default => true,  :window_tokens => 1_000_000, :output_tokens => 32_768, :input_price_for_1m => 2.00,   :output_price_for_1m => 8.00,  :group_size => 75}
+      {:nickname => "gpt-4",        :name => "gpt-4-turbo-2024-04-09", :default => false, :window_tokens => 128_000,   :output_tokens => 4_096,  :input_price_for_1m => 10.00,  :output_price_for_1m => 30.00, :group_size => 200},
+      {:nickname => "gpt-4o-mini",  :name => "gpt-4o-mini",            :default => false, :window_tokens => 128_000,   :output_tokens => 16_384, :input_price_for_1m => 0.15,   :output_price_for_1m => 0.60,  :group_size => 200},
+      {:nickname => "gpt-4o",       :name => "gpt-4o-2024-08-06",      :default => false, :window_tokens => 128_000,   :output_tokens => 16_384, :input_price_for_1m => 2.50,   :output_price_for_1m => 10.00, :group_size => 200},
+      {:nickname => "gpt-4.1-nano", :name => "gpt-4.1-nano",           :default => false, :window_tokens => 1_000_000, :output_tokens => 32_768, :input_price_for_1m => 0.10,   :output_price_for_1m => 0.40,  :group_size => 500},
+      {:nickname => "gpt-4.1-mini", :name => "gpt-4.1-mini",           :default => false, :window_tokens => 1_000_000, :output_tokens => 32_768, :input_price_for_1m => 0.40,   :output_price_for_1m => 1.60,  :group_size => 500},
+      {:nickname => "gpt-4.1",      :name => "gpt-4.1-2025-04-14",     :default => true,  :window_tokens => 1_000_000, :output_tokens => 32_768, :input_price_for_1m => 2.00,   :output_price_for_1m => 8.00,  :group_size => 500}
     ].freeze
   end
 end

data/lib/immosquare-translate/version.rb

@@ -1,3 +1,3 @@
 module ImmosquareTranslate
-  VERSION = "0.1.14".freeze
+  VERSION = "0.1.15".freeze
 end

data/lib/immosquare-translate/yml_translator.rb

@@ -212,7 +212,6 @@ module ImmosquareTranslate
         ##============================================================##
         ## Call OpenAI API
         ##============================================================##
-        index         = 0
         group_size    = model[:group_size]
         from_iso      = ISO_639.find_by_code(from).english_name.split(";").first
         to_iso        = ISO_639.find_by_code(to).english_name.split(";").first
@@ -239,30 +238,6 @@ module ImmosquareTranslate
         }
 
 
-        ##============================================================##
-        ## Estimate the number of window_tokens
-        ## https://platform.openai.com/tokenizer
-        ## English: 75 words => 100 tokens
-        ## French : 55 words => 100 tokens
-        ## ---------
-        ## For each array value we add 5 tokens for the array format.
-        ## [1, "my_word"],
-        ## [  => first token
-        ## 2  => second token
-        ## ,  => third token
-        ## "  => fourth token
-        ## ]" => fifth token
-        ## ---------
-        ## data_open_ai.inspect.size => to get the total number of characters in the array
-        ## with the array structure [""],
-        ##============================================================##
-        estimation_for_100_tokens = from == "fr" ? 55 : 75
-        prompt_tokens_estimation  = (((prompt_system.split.size + prompt_init.split.size + data_open_ai.map {|_index, from| from.split.size }.sum) / estimation_for_100_tokens * 100.0) + (data_open_ai.size * 5)).round
-        split_array               = (prompt_tokens_estimation / model[:window_tokens].to_f).ceil
-        slice_size                = (data_open_ai.size / split_array.to_f).round
-        data_open_ai_sliced       = data_open_ai.each_slice(slice_size).to_a
-
-
         ##============================================================##
         ## Now each slice of the array should no be more than window_tokens
         ## of the model.... We can now translate each slice.
@@ -271,13 +246,18 @@ module ImmosquareTranslate
         ## But it should manage if a word is cut etc...
         ## For the moment we cut it into small group for which we are sure not to exceed the limit
         ##============================================================##
-        puts("fields to translate from #{from_iso} (#{from}) to #{to_iso} (#{to}) : #{data_open_ai.size}#{" by group of #{group_size}" if data_open_ai.size > group_size}")
-        while index < data_open_ai.size
-          data_group = data_open_ai[index, group_size]
+        repeat = (data_open_ai.size / group_size.to_f).ceil
+        puts("fields to translate from #{from_iso} (#{from}) to #{to_iso} (#{to}) : #{data_open_ai.size}#{" by group of #{group_size}" if repeat > 1}")
 
+        repeat.times do |index|
+          index_start = index * group_size
+          index_end   = ((index + 1) * group_size) - 1
+          data_group  = data_open_ai[index_start..index_end]
+
+          next if index > 3
 
           begin
-            puts("call OPENAI Api (with model #{model[:name]}) #{" for #{data_group.size} fields (#{index}-#{index + data_group.size})" if data_open_ai.size > group_size}")
+            puts("call openai api (with model #{model[:nickname]}) #{"for #{data_group.size} fields (#{index_start}-#{index_end})" if repeat > 1}")
             prompt = "#{prompt_init}:\n\n#{data_group.inspect}\n\n"
             body   = {
               :model       => model[:name],
@@ -294,7 +274,6 @@ module ImmosquareTranslate
             puts("responded in #{(Time.now - t0).round(2)} seconds")
             raise(call["error"]["message"]) if call.code != 200
 
-
             ##============================================================##
             ## We check that the result is complete
             ##============================================================##
@@ -324,11 +303,10 @@ module ImmosquareTranslate
               ai_resuslts << [index, translation == cant_be_translated ? nil : translation]
             end
           rescue StandardError => e
-            puts("error OPEN AI API => #{e.message}")
+            puts("error open ai api => #{e.message}")
             puts(e.message)
             puts(e.backtrace)
           end
-          index += group_size
         end
 
 

data/lib/tasks/immosquare-translate.rake

@@ -6,7 +6,7 @@ namespace :immosquare_translate do
   desc "Translate translation files in rails app"
   task :translate_rails_locales => :environment do
     begin
-      source_locale      = ENV.fetch("SOURCE_LOCALE", nil)      || "fr"
+      source_locale      = ENV.fetch("SOURCE_LOCALE", nil)      || I18n.default_locale.to_s
       reset_translations = ENV.fetch("RESET_TRANSLATIONS", nil) || false
       reset_translations = reset_translations == "true"
 
@@ -14,12 +14,19 @@ namespace :immosquare_translate do
       raise("Please provide a valid boolean for reset_translations") if ![true, false].include?(reset_translations)
 
       locales = I18n.available_locales.map(&:to_s).reject {|l| l == source_locale }
-      puts("Translating from #{source_locale} to #{locales.join(", ")} with reset_translations=#{reset_translations}")
+      return puts("Any translation asked") if locales.empty?
+
+      puts("Translations asked :")
+      locales.each do |locale|
+        puts("#{source_locale} => #{locale}")
+      end
+
       Dir.glob("#{Rails.root}/config/locales/**/*#{source_locale}.yml").each do |file|
         locales.each do |locale|
           ImmosquareTranslate::YmlTranslator.translate(file, locale, :reset_translations => reset_translations)
         end
       end
+      puts("Translations done")
     rescue StandardError => e
       puts(e.message)
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: immosquare-translate
 version: !ruby/object:Gem::Version
-  version: 0.1.14
+  version: 0.1.15
 platform: ruby
 authors:
 - immosquare