immosquare-yaml 0.1.25 → 0.1.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4404e071795297a64e93c211cf7f45794894b940b2f14c02b96a9771a9fc5740
-  data.tar.gz: bfc66cb17d38c5dce224adde7c97902c7ae61d68687e476d27c40a19a61f420f
+  metadata.gz: de0be4bb7791b8644c8ac692d1d15ec2632ca8b026cf834bca21fcdfe265c469
+  data.tar.gz: 14df18840e34827289a76adcbfdde4aa96e6a0ade2f6e0b4913feb97451b92f0
 SHA512:
-  metadata.gz: 3e9e26bc45f59275a765ace0f1cffde57b570d52f6ee49b973c5d861a7a3d0ec13730255e364aed902040875aa0a2435333489a377dbed6f4c10601bc44c4c4d
-  data.tar.gz: 91a45dfe6c5e8debd0157a6aa66e78d1caea336a1695203051dfa7577df7a8343a47a859b8560ca23f837cdf8c4942816e578ed25d8deca782360f01c0384dea
+  metadata.gz: eb8f99e6579320e5760c080383cdf3f6c0a828a90eb2e65f756fc14680e1262d96034e4dc87bc48d56c7f4307015962dcb615e1ff6b1e51ee69155328cf05644
+  data.tar.gz: 0c042f2c93d8e68adf9c1d3680b48a1c7c575b65cb109533c238b00e72f7567ca15c444eba24361ded63984f08c72d4b541296395d2f382402127c042a136975

data/lib/immosquare-yaml/version.rb

@@ -1,3 +1,3 @@
 module ImmosquareYaml
-  VERSION = "0.1.25".freeze
+  VERSION = "0.1.27".freeze
 end

data/lib/immosquare-yaml.rb

@@ -1,9 +1,9 @@
 require          "English"
 require          "psych"
+require          "date"
 require          "immosquare-extensions"
 require_relative "immosquare-yaml/configuration"
 require_relative "immosquare-yaml/shared_methods"
-require_relative "immosquare-yaml/translate"
 require_relative "immosquare-yaml/railtie" if defined?(Rails)
 
 ##===========================================================================##
@@ -626,6 +626,17 @@ module ImmosquareYaml
                                     value.start_with?(SPACE) ||
                                     value.end_with?(SPACE)
         end
+
+        ##============================================================##
+        ## Final clean to prevent
+        ## "yes": YES
+        ## "no": NO
+        ##============================================================##
+        value = "\"#{value}\"" if RESERVED_KEYS.include?(value)
+
+        ##============================================================##
+        ## Return the cleaned value
+        ##============================================================##
         value
       end
       is_array.instance_of?(String) ? values.first : "[#{values.join(", ")}]"

data/lib/tasks/immosquare-yaml.rake

@@ -1,30 +1,4 @@
 namespace :immosquare_yaml do
-  ##============================================================##
-  ## Function to translate translation files in rails app
-  ## rake immosquare_yaml:translate SOURCE_LOCALE=fr
-  ##============================================================##
-  desc "Translate translation files in rails app"
-  task :translate => :environment do
-    begin
-      source_locale      = ENV.fetch("SOURCE_LOCALE", nil) || "fr"
-      reset_translations = ENV.fetch("RESET_TRANSLATIONS", nil) || false
-      reset_translations = reset_translations == "true"
-
-      raise("Please provide a valid locale")                         if !I18n.available_locales.map(&:to_s).include?(source_locale)
-      raise("Please provide a valid boolean for reset_translations") if ![true, false].include?(reset_translations)
-
-      locales = I18n.available_locales.map(&:to_s).reject {|l| l == source_locale }
-      puts("Translating from #{source_locale} to #{locales.join(", ")} with reset_translations=#{reset_translations}")
-      Dir.glob("#{Rails.root}/config/locales/**/*#{source_locale}.yml").each do |file|
-        locales.each do |locale|
-          ImmosquareYaml::Translate.translate(file, locale, :reset_translations => reset_translations)
-        end
-      end
-    rescue StandardError => e
-      puts(e.message)
-    end
-  end
-
   ##============================================================##
   ## Function to clean translation files in rails app
   ##============================================================##

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: immosquare-yaml
 version: !ruby/object:Gem::Version
-  version: 0.1.25
+  version: 0.1.27
 platform: ruby
 authors:
-- IMMO SQUARE
+- immosquare
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-15 00:00:00.000000000 Z
+date: 2024-11-28 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: httparty
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: immosquare-extensions
   requirement: !ruby/object:Gem::Requirement
@@ -33,7 +19,7 @@ dependencies:
         version: '0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.14
+        version: 0.1.18
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -43,21 +29,7 @@ dependencies:
         version: '0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.14
-- !ruby/object:Gem::Dependency
-  name: iso-639
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.18
 description: IMMOSQUARE-YAML is a specialized Ruby gem tailored primarily for parsing
   and dumping YML translation files, addressing challenges faced with other parsers
   like interpreting translation keys as booleans, multi-line strings, and more.
@@ -71,10 +43,9 @@ files:
 - lib/immosquare-yaml/configuration.rb
 - lib/immosquare-yaml/railtie.rb
 - lib/immosquare-yaml/shared_methods.rb
-- lib/immosquare-yaml/translate.rb
 - lib/immosquare-yaml/version.rb
 - lib/tasks/immosquare-yaml.rake
-homepage: https://github.com/IMMOSQUARE/immosquare-yaml
+homepage: https://github.com/immosquare/immosquare-yaml
 licenses:
 - MIT
 metadata: {}
@@ -93,7 +64,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.13
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: A YAML parser optimized for translation files.

data/lib/immosquare-yaml/translate.rb

@@ -1,365 +0,0 @@
-require "iso-639"
-require "httparty"
-
-
-module ImmosquareYaml
-  module Translate
-    extend SharedMethods
-
-    class << self
-
-      def translate(file_path, locale_to,  options = {})
-        begin
-          ##=============================================================##
-          ## options
-          ##=============================================================##
-          options = {
-            :reset_translations => false
-          }.merge(options)
-          options[:reset_translations] = false  if ![true, false].include?(options[:reset_translations])
-
-
-          ##=============================================================##
-          ## Load config keys from config_dev.yml
-          ##=============================================================##
-          raise("Error: openai_api_key not found in config_dev.yml") if ImmosquareYaml.configuration.openai_api_key.nil?
-          raise("Error: File #{file_path} not found")                if !File.exist?(file_path)
-          raise("Error: locale is not a locale")                     if !locale_to.is_a?(String) || locale_to.size != 2
-
-          ##============================================================##
-          ## We clean the file before translation
-          ##============================================================##
-          ImmosquareYaml.clean(file_path)
-
-          ##============================================================##
-          ## We parse the clean input file
-          ##============================================================##
-          hash_from = ImmosquareYaml.parse(file_path)
-          raise("#{file_path} is not a correct yml translation file") if !hash_from.is_a?(Hash) && hash_from.keys.size > 1
-
-          ##============================================================##
-          ## Check if the locale is present in the file
-          ##============================================================##
-          locale_from = hash_from.keys.first.to_s
-          raise("Error: The destination file (#{locale_to}) is the same as the source file (#{locale_from}).")    if locale_from == locale_to
-          raise("Error: Expected the source file (#{file_path}) to end with '#{locale_from}.yml' but it didn't.") if !file_path.end_with?("#{locale_from}.yml")
-
-
-          ##============================================================##
-          ## Prepare the output file
-          ##============================================================##
-          file_basename        = File.basename(file_path)
-          file_dirname         = File.dirname(file_path)
-          translated_file_path = "#{file_dirname}/#{file_basename.gsub("#{locale_from}.yml", "#{locale_to}.yml")}"
-
-          ##============================================================##
-          ## We create a hash with all keys from the source file
-          ##============================================================##
-          hash_to = {locale_to => hash_from.delete(locale_from)}
-
-          ##============================================================##
-          ## We create a array with all keys from the source file
-          ##============================================================##
-          array_to = translatable_array(hash_to)
-          array_to = array_to.map {|k, v| [k, v, nil] }
-
-          ##============================================================##
-          ## If we already have a translation file for the language
-          ## we get the values in it and put it in our
-          ## file... You have to do well with !nil?
-          ## to retrieve the values "" and " "...
-          ##============================================================##
-          if File.exist?(translated_file_path) && options[:reset_translations] == false
-            temp_hash = ImmosquareYaml.parse(translated_file_path)
-            raise("#{translated_file_path} is not a correct yml translation file") if !temp_hash.is_a?(Hash) && temp_hash.keys.size > 1
-
-            ##============================================================##
-            ## t can be nil if the key is not present in the source file
-            ##============================================================##
-            translatable_array(temp_hash).each do |key, value|
-              t    = array_to.find {|k, _v| k == key }
-              t[2] = value if !t.nil? && !value.nil?
-            end
-          end
-
-          ##============================================================##
-          ## Here we have to do all the translation logic...
-          ## For the moment we use the OPENAI API, but we can imagine
-          ## using other translation APIs in the future.
-          ##============================================================##
-          translated_array = translate_with_open_ai(array_to, locale_from, locale_to)
-
-          ##============================================================##
-          ## Then we have to reformat the output yml file
-          ##============================================================##
-          final_array = translated_array.map do |k, _from, to|
-            parsed_to = !to.nil? && to.start_with?("[") && to.end_with?("]") ? JSON.parse(to) : to
-            [k, parsed_to]
-          end
-          final_hash = translatable_hash(final_array)
-
-
-          ##============================================================##
-          ## We write the output file and clean it
-          ##============================================================##
-          File.write(translated_file_path, ImmosquareYaml.dump(final_hash))
-          ImmosquareYaml.clean(translated_file_path)
-        rescue StandardError => e
-          puts(e.message)
-          puts(e.backtrace)
-          false
-        end
-      end
-
-
-      private
-
-      ##============================================================##
-      ## To translatable hash to array
-      ## opitons are :
-      ## :format    => "string" or "array"
-      ## :keys_only => true or false
-      ## {:fr=>{"demo1"=>"demo1", "demo2"=>{"demo2-1"=>"demo2-1"}}}
-      ## format = "string" and keys_only = false => [["fr.demo1", "demo1"], ["fr.demo2.demo2-1", "demo2-1"]]
-      ## format = "string" and keys_only = true  => ["fr.demo1", "fr.demo2.demo2-1"]
-      ## format = "array"  and keys_only = false => [[["fr", "demo1"], "demo1"], [["fr", "demo2", "demo2-1"], "demo2-1"]]
-      ## format = "array"  and keys_only = true  => [["fr", "demo1"], ["fr", "demo2", "demo2-1"]]
-      ## ============================================================
-      def translatable_array(hash, key = nil, result = [], **options)
-        options = {
-          :format    => "string",
-          :keys_only => false
-        }.merge(options)
-        options[:keys_only] = false    if ![true, false].include?(options[:keys_only])
-        options[:format]    = "string" if !["string", "array"].include?(options[:format])
-
-
-        if hash.is_a?(Hash)
-          hash.each_key do |k|
-            translatable_array(hash[k], "#{key}#{":" if !key.nil?}#{k}", result, **options)
-          end
-        else
-          r2 = options[:format] == "string" ? key.split(":").join(".") : key.split(":")
-          result << (options[:keys_only] ? r2 : [r2, hash.is_a?(Array) ? hash.to_json : hash])
-        end
-        result
-      end
-
-      ##============================================================##
-      ## We can do the inverse of the previous function
-      ##============================================================##
-      def translatable_hash(array)
-        data_hash = array.to_h
-        final     = {}
-        data_hash.each do |key, value|
-          key_parts     = key.split(".")
-          leaf          = key_parts.pop
-          parent        = key_parts.inject(final) {|h, k| h[k] ||= {} }
-          parent[leaf]  = value
-        end
-        final
-      end
-
-      ##============================================================##
-      ## Translate with OpenAI
-      ##
-      ## [
-      ##  ["en.mlsconnect.contact_us", "Nous contacter", "Contact us"],
-      ##  ["en.mlsconnect.description", "Description", nil],
-      ##  ...
-      ## ]
-      ##============================================================##
-      def translate_with_open_ai(array, from, to)
-        ##============================================================##
-        ## https://platform.openai.com/docs/models/
-        ## https://openai.com/pricing
-        ##============================================================##
-        model_name = ImmosquareYaml.configuration.openai_model
-        models     = [
-          {:name => "gpt-3.5-turbo-0125", :window_tokens => 16_385,  :output_tokens => 4096, :input_price_for_1m => 0.50,  :output_price_for_1m => 1.50,  :group_size => 75},
-          {:name => "gpt-4-0125-preview", :window_tokens => 128_000, :output_tokens => 4096, :input_price_for_1m => 10.00, :output_price_for_1m => 30.00, :group_size => 75}
-        ]
-        model = models.find {|m| m[:name] == model_name }
-        model = models.find {|m| m[:name] == "gpt-4-0125-preview" } if model.nil?
-
-        ##============================================================##
-        ## Manage blank values
-        ##============================================================##
-        blank_values       = [NOTHING, SPACE, "\"\"", "\"#{SPACE}\""]
-        cant_be_translated = "CANNOT-BE-TRANSLATED"
-        array              = array.map do |key, from, to|
-          [key, from, blank_values.include?(from) ? from : to]
-        end
-
-
-        ##============================================================##
-        ## we want to send as little data as possible to openAI because
-        ## we pay for the volume of data sent. So we're going to send. We put
-        ## a number rather than a string for the translations to be made.
-        ## --------
-        ## Remove the translations that have already been made
-        ##============================================================##
-        data_open_ai = array.clone
-        data_open_ai = data_open_ai.map.with_index {|(_k, from, to), index| [index, from, to] }
-        data_open_ai = data_open_ai.select {|_index, from, to| !from.nil? && to.nil? }
-
-        ##============================================================##
-        ## Remove quotes surrounding the value if they are present.
-        ## and remove to to avoid error in translation
-        ##============================================================##
-        data_open_ai = data_open_ai.map do |index, from, _to|
-          from = from.to_s
-          from = from[1..-2] while (from.start_with?(DOUBLE_QUOTE) && from.end_with?(DOUBLE_QUOTE)) || (from.start_with?(SIMPLE_QUOTE) && from.end_with?(SIMPLE_QUOTE))
-          [index, from]
-        end
-
-        return array if data_open_ai.empty?
-
-
-        ##============================================================##
-        ## Call OpenAI API
-        ##============================================================##
-        index         = 0
-        group_size    = model[:group_size]
-        from_iso      = ISO_639.find_by_code(from).english_name.split(";").first
-        to_iso        = ISO_639.find_by_code(to).english_name.split(";").first
-        ai_resuslts   = []
-        prompt_system = "You are a translation tool from #{from_iso} to #{to_iso}\n" \
-                        "The input is an array of pairs, where each pair contains an index and a string to translate, formatted as [index, string_to_translate]\n" \
-                        "Your task is to create an output ARRAY where each element is a pair consisting of the index and the translated string, formatted as [index, 'string_translated']\n" \
-                        "If a string_to_translate starts with [ and ends with ], it is considered a special string that should be treated as a JSON object. Otherwise, it's a normal string.\n" \
-                        "\nRules to respect for JSON objects:\n" \
-                        "- You need to translate ONLY the values of the JSON object, not the keys. Do not change anything in the format, just translate the values.\n" \
-                        "- Respect all following rules for normal strings to translate the values\n" \
-                        "\nRules to respect for normal strings:\n" \
-                        "- Do not escape apostrophes in translated strings; leave them as they are.\n" \
-                        "- Special characters, except apostrophes, that need to be escaped in translated strings should be escaped using a single backslash (\\), not double (\\\\).\n" \
-                        "- If a string cannot be translated use the string '#{cant_be_translated}' translated as the translation value witouth quote (simple or double) quote, just the string\n" \
-                        "- If you dont know the correct translatation use the #{cant_be_translated} strategy of the preceding point\n" \
-                        "- Use only double quotes (\") to enclose translated strings and avoid using single quotes (').\n" \
-                        "- Your output must ONLY be an array with the same number of pairs as the input, without any additional text or explanation. DO NOT COMMENT!\n" \
-                        "- You need to check that the globle array is correctly closed at the end of the response. (the response must therefore end with ]] to to be consistent)"
-        prompt_init   = "Please proceed with translating the following array:"
-        headers       = {
-          "Content-Type"  => "application/json",
-          "Authorization" => "Bearer #{ImmosquareYaml.configuration.openai_api_key}"
-        }
-
-
-        ##============================================================##
-        ## Estimate the number of window_tokens
-        ## https://platform.openai.com/tokenizer
-        ## English: 75 words => 100 tokens
-        ## French : 55 words => 100 tokens
-        ## -----------------
-        ## For each array value we add 5 tokens for the array format.
-        ## [1, "my_word"],
-        ## [  => first token
-        ## 2  => second token
-        ## ,  => third token
-        ## "  => fourth token
-        ## ]" => fifth token
-        ## -----------------
-        # data_open_ai.inspect.size => to get the total number of characters in the array
-        ## with the array structure [""],
-        ##============================================================##
-        estimation_for_100_tokens = from == "fr" ? 55 : 75
-        prompt_tokens_estimation  = (((prompt_system.split.size + prompt_init.split.size + data_open_ai.map {|_index, from| from.split.size }.sum) / estimation_for_100_tokens * 100.0) + (data_open_ai.size * 5)).round
-        split_array               = (prompt_tokens_estimation / model[:window_tokens].to_f).ceil
-        slice_size                = (data_open_ai.size / split_array.to_f).round
-        data_open_ai_sliced       = data_open_ai.each_slice(slice_size).to_a
-
-
-        ##============================================================##
-        ## Now each slice of the array should no be more than window_tokens
-        ## of the model.... We can now translate each slice.
-        ## ---------------------------------
-        ## Normally we could send the whole slice at once and tell the api to continue if its response is not tarnished...
-        ## But it should manage if a word is cut etc...
-        ## For the moment we cut it into small group for which we are sure not to exceed the limit
-        ##============================================================##
-        puts("fields to translate from #{from_iso} (#{from}) to #{to_iso} (#{to}) : #{data_open_ai.size}#{" by group of #{group_size}" if data_open_ai.size > group_size}")
-        while index < data_open_ai.size
-          data_group = data_open_ai[index, group_size]
-
-
-          begin
-            puts("call OPENAI Api (with model #{model[:name]}) #{" for #{data_group.size} fields (#{index}-#{index + data_group.size})" if data_open_ai.size > group_size}")
-            prompt = "#{prompt_init}:\n\n#{data_group.inspect}\n\n"
-            body   = {
-              :model       => model[:name],
-              :messages    => [
-                {:role => "system", :content => prompt_system},
-                {:role => "user",   :content => prompt}
-              ],
-              :temperature => 0.0
-            }
-            t0   = Time.now
-            call = HTTParty.post("https://api.openai.com/v1/chat/completions", :body => body.to_json, :headers => headers, :timeout => 500)
-
-            puts("responded in #{(Time.now - t0).round(2)} seconds")
-            raise(call["error"]["message"]) if call.code != 200
-
-
-            ##============================================================##
-            ## We check that the result is complete
-            ##============================================================##
-            response  = JSON.parse(call.body)
-            choice    = response["choices"][0]
-            raise("Result is not complete") if choice["finish_reason"] != "stop"
-
-
-            ##============================================================##
-            ## We calculate the estimate price of the call
-            ##============================================================##
-            input_price   = response["usage"]["prompt_tokens"]     * (model[:input_price_for_1m] / 1_000_000)
-            output_price  = response["usage"]["completion_tokens"] * (model[:output_price_for_1m] / 1_000_000)
-            price         = input_price + output_price
-            puts("Estimate price => #{input_price.round(3)} + #{output_price.round(3)} = #{price.round(3)} USD")
-
-            ##============================================================##
-            ## We check that the result is an array
-            ##============================================================##
-            content = eval(choice["message"]["content"])
-            raise("Is not an array") if !content.is_a?(Array)
-
-            ##============================================================##
-            ## We save the result
-            ##============================================================##
-            content.each do |index, translation|
-              ai_resuslts << [index, translation == cant_be_translated ? nil : translation]
-            end
-          rescue StandardError => e
-            puts("error OPEN AI API => #{e.message}")
-            puts(e.message)
-            puts(e.backtrace)
-          end
-          index += group_size
-        end
-
-
-        ##============================================================##
-        ## We put the translations in the original array
-        ##============================================================##
-        ai_resuslts.each do |index, translation|
-          begin
-            array[index.to_i][2] = translation
-          rescue StandardError => e
-            puts(e.message)
-          end
-        end
-
-        ##============================================================##
-        ## We return the modified array
-        ##============================================================##
-        array.map.with_index do |(k, from, to), index|
-          from = from.to_s
-          to   = "#{DOUBLE_QUOTE}#{to}#{DOUBLE_QUOTE}" if ai_resuslts.find {|i, _t| i == index } && ((from.start_with?(DOUBLE_QUOTE) && from.end_with?(DOUBLE_QUOTE)) || (from.start_with?(SIMPLE_QUOTE) && from.end_with?(SIMPLE_QUOTE)))
-          [k, from, to]
-        end
-      end
-
-
-    end
-  end
-end