immosquare-translate 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/immosquare-translate/configuration.rb +12 -0
- data/lib/immosquare-translate/shared_methods.rb +14 -0
- data/lib/immosquare-translate/translator.rb +88 -0
- data/lib/immosquare-translate/version.rb +3 -0
- data/lib/immosquare-translate/yml_translator.rb +361 -0
- data/lib/immosquare-translate.rb +30 -0
- data/lib/tasks/immosquare-yaml.rake +37 -0
- metadata +103 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: fb9bf1291f2d54448b290ba1cc3514c32fba97a989cdaa3b9d7b30950642e3c5
|
4
|
+
data.tar.gz: d7118588f2c537dae3b9df764a6217a73bedec70d68f2260f393e399d84aac8e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 4859e9d1d6b6c3a2600aba4fab39b41e1deb0ad19915a96e629fc77afb91ee5ecf56778daa89c3c90af02ff811379a235755abdb1eb59353622fd39b1629bb5a
|
7
|
+
data.tar.gz: 4e912e82473338dec1f10f868f11bd88a0a3a0ee63471b713c28ccbbad53770d35c826ecd4549c2e868fc55a003abf0b20947a204ab71b501a290e83884bba15
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module ImmosquareTranslate
|
2
|
+
module SharedMethods
|
3
|
+
NOTHING = "".freeze
|
4
|
+
SPACE = " ".freeze
|
5
|
+
SIMPLE_QUOTE = "'".freeze
|
6
|
+
DOUBLE_QUOTE = '"'.freeze
|
7
|
+
|
8
|
+
|
9
|
+
OPEN_AI_MODELS = [
|
10
|
+
{:name => "gpt-3.5-turbo-0125", :window_tokens => 16_385, :output_tokens => 4096, :input_price_for_1m => 0.50, :output_price_for_1m => 1.50, :group_size => 75},
|
11
|
+
{:name => "gpt-4-0125-preview", :window_tokens => 128_000, :output_tokens => 4096, :input_price_for_1m => 10.00, :output_price_for_1m => 30.00, :group_size => 75}
|
12
|
+
].freeze
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
module ImmosquareTranslate
|
2
|
+
module Translator
|
3
|
+
extend SharedMethods
|
4
|
+
class << self
|
5
|
+
|
6
|
+
##============================================================##
|
7
|
+
## Translate Data
|
8
|
+
## ["Bonjour"], "fr", ["en", "es", "it"]
|
9
|
+
## text : array
|
10
|
+
## from : string
|
11
|
+
## to : array
|
12
|
+
##============================================================##
|
13
|
+
def translate(text, from, to)
|
14
|
+
begin
|
15
|
+
raise("Error: openai_api_key not found in config_dev.yml") if ImmosquareTranslate.configuration.openai_api_key.nil?
|
16
|
+
raise("Error: locale from is not a locale") if !from.is_a?(String) || from.size != 2
|
17
|
+
raise("Error: locales is not an array of locales") if !to.is_a?(Array) || to.empty? || to.any? {|l| !l.is_a?(String) || l.size != 2 }
|
18
|
+
|
19
|
+
model_name = ImmosquareYaml.configuration.openai_model
|
20
|
+
model = OPEN_AI_MODELS.find {|m| m[:name] == model_name }
|
21
|
+
model = OPEN_AI_MODELS.find {|m| m[:name] == "gpt-4-0125-preview" } if model.nil?
|
22
|
+
from_iso = ISO_639.find_by_code(from).english_name.split(";").first
|
23
|
+
to_iso = to.map {|l| ISO_639.find_by_code(l).english_name.split(";").first }
|
24
|
+
headers = {
|
25
|
+
"Content-Type" => "application/json",
|
26
|
+
"Authorization" => "Bearer #{ImmosquareTranslate.configuration.openai_api_key}"
|
27
|
+
}
|
28
|
+
|
29
|
+
prompt_system = "As a sophisticated translation AI, your role is to translate sentences from a specified source language to multiple target languages. " \
|
30
|
+
"It is imperative that you return the translations in a single, pure JSON string format. Use ISO 639-1 language codes for specifying languages. " \
|
31
|
+
"Ensure that the output does not include markdown or any other formatting characters. Adhere to the JSON structure meticulously."
|
32
|
+
|
33
|
+
|
34
|
+
prompt = "Translate the following sentences from '#{from_iso}' into the languages #{to_iso.join(", ")}, and format the output as a single, pure JSON string. " \
|
35
|
+
"Follow the structure: {\"datas\":[{\"en\":\"English Translation\",\"es\":\"Spanish Translation\",\"it\":\"Italian Translation\"}]}, using the correct ISO 639-1 language codes for each translation. " \
|
36
|
+
"Your response should strictly conform to this JSON structure without any additional characters or formatting. Sentences to translate are:"
|
37
|
+
|
38
|
+
text.each_with_index do |sentence, index|
|
39
|
+
prompt += "\n#{index + 1}: #{sentence}"
|
40
|
+
end
|
41
|
+
|
42
|
+
|
43
|
+
body = {
|
44
|
+
:model => model[:name],
|
45
|
+
:messages => [
|
46
|
+
{:role => "system", :content => prompt_system},
|
47
|
+
{:role => "user", :content => prompt}
|
48
|
+
],
|
49
|
+
:temperature => 0.0
|
50
|
+
}
|
51
|
+
|
52
|
+
|
53
|
+
t0 = Time.now
|
54
|
+
call = HTTParty.post("https://api.openai.com/v1/chat/completions", :body => body.to_json, :headers => headers, :timeout => 500)
|
55
|
+
|
56
|
+
|
57
|
+
puts("responded in #{(Time.now - t0).round(2)} seconds")
|
58
|
+
raise(call["error"]["message"]) if call.code != 200
|
59
|
+
|
60
|
+
##============================================================##
|
61
|
+
## We check that the result is complete
|
62
|
+
##============================================================##
|
63
|
+
response = JSON.parse(call.body)
|
64
|
+
choice = response["choices"][0]
|
65
|
+
raise("Result is not complete") if choice["finish_reason"] != "stop"
|
66
|
+
|
67
|
+
##============================================================##
|
68
|
+
## We calculate the estimate price of the call
|
69
|
+
##============================================================##
|
70
|
+
input_price = response["usage"]["prompt_tokens"] * (model[:input_price_for_1m] / 1_000_000)
|
71
|
+
output_price = response["usage"]["completion_tokens"] * (model[:output_price_for_1m] / 1_000_000)
|
72
|
+
price = input_price + output_price
|
73
|
+
puts("Estimate price => #{input_price.round(3)} + #{output_price.round(3)} = #{price.round(3)} USD")
|
74
|
+
|
75
|
+
|
76
|
+
p = JSON.parse(choice["message"]["content"])
|
77
|
+
p["datas"]
|
78
|
+
rescue StandardError => e
|
79
|
+
puts(e.message)
|
80
|
+
puts(e.backtrace)
|
81
|
+
false
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,361 @@
|
|
1
|
+
require "iso-639"
|
2
|
+
require "httparty"
|
3
|
+
require "immosquare-yaml"
|
4
|
+
|
5
|
+
module ImmosquareTranslate
|
6
|
+
module YmlTranslator
|
7
|
+
extend SharedMethods
|
8
|
+
|
9
|
+
class << self
|
10
|
+
|
11
|
+
def translate(file_path, locale_to, options = {})
|
12
|
+
begin
|
13
|
+
##=============================================================##
|
14
|
+
## options
|
15
|
+
##=============================================================##
|
16
|
+
options = {
|
17
|
+
:reset_translations => false
|
18
|
+
}.merge(options)
|
19
|
+
options[:reset_translations] = false if ![true, false].include?(options[:reset_translations])
|
20
|
+
|
21
|
+
|
22
|
+
##=============================================================##
|
23
|
+
## Load config keys from config_dev.yml
|
24
|
+
##=============================================================##
|
25
|
+
raise("Error: openai_api_key not found in config_dev.yml") if ImmosquareTranslate.configuration.openai_api_key.nil?
|
26
|
+
raise("Error: File #{file_path} not found") if !File.exist?(file_path)
|
27
|
+
raise("Error: locale is not a locale") if !locale_to.is_a?(String) || locale_to.size != 2
|
28
|
+
|
29
|
+
##============================================================##
|
30
|
+
## We clean the file before translation
|
31
|
+
##============================================================##
|
32
|
+
ImmosquareYaml.clean(file_path)
|
33
|
+
|
34
|
+
##============================================================##
|
35
|
+
## We parse the clean input file
|
36
|
+
##============================================================##
|
37
|
+
hash_from = ImmosquareYaml.parse(file_path)
|
38
|
+
raise("#{file_path} is not a correct yml translation file") if !hash_from.is_a?(Hash) && hash_from.keys.size > 1
|
39
|
+
|
40
|
+
##============================================================##
|
41
|
+
## Check if the locale is present in the file
|
42
|
+
##============================================================##
|
43
|
+
locale_from = hash_from.keys.first.to_s
|
44
|
+
raise("Error: The destination file (#{locale_to}) is the same as the source file (#{locale_from}).") if locale_from == locale_to
|
45
|
+
raise("Error: Expected the source file (#{file_path}) to end with '#{locale_from}.yml' but it didn't.") if !file_path.end_with?("#{locale_from}.yml")
|
46
|
+
|
47
|
+
|
48
|
+
##============================================================##
|
49
|
+
## Prepare the output file
|
50
|
+
##============================================================##
|
51
|
+
file_basename = File.basename(file_path)
|
52
|
+
file_dirname = File.dirname(file_path)
|
53
|
+
translated_file_path = "#{file_dirname}/#{file_basename.gsub("#{locale_from}.yml", "#{locale_to}.yml")}"
|
54
|
+
|
55
|
+
##============================================================##
|
56
|
+
## We create a hash with all keys from the source file
|
57
|
+
##============================================================##
|
58
|
+
hash_to = {locale_to => hash_from.delete(locale_from)}
|
59
|
+
|
60
|
+
##============================================================##
|
61
|
+
## We create a array with all keys from the source file
|
62
|
+
##============================================================##
|
63
|
+
array_to = translatable_array(hash_to)
|
64
|
+
array_to = array_to.map {|k, v| [k, v, nil] }
|
65
|
+
|
66
|
+
##============================================================##
|
67
|
+
## If we already have a translation file for the language
|
68
|
+
## we get the values in it and put it in our
|
69
|
+
## file... You have to do well with !nil?
|
70
|
+
## to retrieve the values "" and " "...
|
71
|
+
##============================================================##
|
72
|
+
if File.exist?(translated_file_path) && options[:reset_translations] == false
|
73
|
+
temp_hash = ImmosquareYaml.parse(translated_file_path)
|
74
|
+
raise("#{translated_file_path} is not a correct yml translation file") if !temp_hash.is_a?(Hash) && temp_hash.keys.size > 1
|
75
|
+
|
76
|
+
##============================================================##
|
77
|
+
## t can be nil if the key is not present in the source file
|
78
|
+
##============================================================##
|
79
|
+
translatable_array(temp_hash).each do |key, value|
|
80
|
+
t = array_to.find {|k, _v| k == key }
|
81
|
+
t[2] = value if !t.nil? && !value.nil?
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
##============================================================##
|
86
|
+
## Here we have to do all the translation logic...
|
87
|
+
## For the moment we use the OPENAI API, but we can imagine
|
88
|
+
## using other translation APIs in the future.
|
89
|
+
##============================================================##
|
90
|
+
translated_array = translate_with_open_ai(array_to, locale_from, locale_to)
|
91
|
+
|
92
|
+
##============================================================##
|
93
|
+
## Then we have to reformat the output yml file
|
94
|
+
##============================================================##
|
95
|
+
final_array = translated_array.map do |k, _from, to|
|
96
|
+
parsed_to = !to.nil? && to.start_with?("[") && to.end_with?("]") ? JSON.parse(to) : to
|
97
|
+
[k, parsed_to]
|
98
|
+
end
|
99
|
+
final_hash = translatable_hash(final_array)
|
100
|
+
|
101
|
+
|
102
|
+
##============================================================##
|
103
|
+
## We write the output file and clean it
|
104
|
+
##============================================================##
|
105
|
+
File.write(translated_file_path, ImmosquareYaml.dump(final_hash))
|
106
|
+
ImmosquareYaml.clean(translated_file_path)
|
107
|
+
rescue StandardError => e
|
108
|
+
puts(e.message)
|
109
|
+
puts(e.backtrace)
|
110
|
+
false
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
##============================================================##
|
118
|
+
## To translatable hash to array
|
119
|
+
## opitons are :
|
120
|
+
## :format => "string" or "array"
|
121
|
+
## :keys_only => true or false
|
122
|
+
## {:fr=>{"demo1"=>"demo1", "demo2"=>{"demo2-1"=>"demo2-1"}}}
|
123
|
+
## format = "string" and keys_only = false => [["fr.demo1", "demo1"], ["fr.demo2.demo2-1", "demo2-1"]]
|
124
|
+
## format = "string" and keys_only = true => ["fr.demo1", "fr.demo2.demo2-1"]
|
125
|
+
## format = "array" and keys_only = false => [[["fr", "demo1"], "demo1"], [["fr", "demo2", "demo2-1"], "demo2-1"]]
|
126
|
+
## format = "array" and keys_only = true => [["fr", "demo1"], ["fr", "demo2", "demo2-1"]]
|
127
|
+
## ============================================================
|
128
|
+
def translatable_array(hash, key = nil, result = [], **options)
|
129
|
+
options = {
|
130
|
+
:format => "string",
|
131
|
+
:keys_only => false
|
132
|
+
}.merge(options)
|
133
|
+
options[:keys_only] = false if ![true, false].include?(options[:keys_only])
|
134
|
+
options[:format] = "string" if !["string", "array"].include?(options[:format])
|
135
|
+
|
136
|
+
|
137
|
+
if hash.is_a?(Hash)
|
138
|
+
hash.each_key do |k|
|
139
|
+
translatable_array(hash[k], "#{key}#{":" if !key.nil?}#{k}", result, **options)
|
140
|
+
end
|
141
|
+
else
|
142
|
+
r2 = options[:format] == "string" ? key.split(":").join(".") : key.split(":")
|
143
|
+
result << (options[:keys_only] ? r2 : [r2, hash.is_a?(Array) ? hash.to_json : hash])
|
144
|
+
end
|
145
|
+
result
|
146
|
+
end
|
147
|
+
|
148
|
+
##============================================================##
|
149
|
+
## We can do the inverse of the previous function
|
150
|
+
##============================================================##
|
151
|
+
def translatable_hash(array)
|
152
|
+
data_hash = array.to_h
|
153
|
+
final = {}
|
154
|
+
data_hash.each do |key, value|
|
155
|
+
key_parts = key.split(".")
|
156
|
+
leaf = key_parts.pop
|
157
|
+
parent = key_parts.inject(final) {|h, k| h[k] ||= {} }
|
158
|
+
parent[leaf] = value
|
159
|
+
end
|
160
|
+
final
|
161
|
+
end
|
162
|
+
|
163
|
+
##============================================================##
|
164
|
+
## Translate with OpenAI
|
165
|
+
##
|
166
|
+
## [
|
167
|
+
## ["en.mlsconnect.contact_us", "Nous contacter", "Contact us"],
|
168
|
+
## ["en.mlsconnect.description", "Description", nil],
|
169
|
+
## ...
|
170
|
+
## ]
|
171
|
+
##============================================================##
|
172
|
+
def translate_with_open_ai(array, from, to)
|
173
|
+
##============================================================##
|
174
|
+
## https://platform.openai.com/docs/models/
|
175
|
+
## https://openai.com/pricing
|
176
|
+
##============================================================##
|
177
|
+
model_name = ImmosquareYaml.configuration.openai_model
|
178
|
+
model = OPEN_AI_MODELS.find {|m| m[:name] == model_name }
|
179
|
+
model = OPEN_AI_MODELS.find {|m| m[:name] == "gpt-4-0125-preview" } if model.nil?
|
180
|
+
|
181
|
+
##============================================================##
|
182
|
+
## Manage blank values
|
183
|
+
##============================================================##
|
184
|
+
blank_values = [NOTHING, SPACE, "\"\"", "\"#{SPACE}\""]
|
185
|
+
cant_be_translated = "CANNOT-BE-TRANSLATED"
|
186
|
+
array = array.map do |key, from, to|
|
187
|
+
[key, from, blank_values.include?(from) ? from : to]
|
188
|
+
end
|
189
|
+
|
190
|
+
|
191
|
+
##============================================================##
|
192
|
+
## we want to send as little data as possible to openAI because
|
193
|
+
## we pay for the volume of data sent. So we're going to send. We put
|
194
|
+
## a number rather than a string for the translations to be made.
|
195
|
+
## --------
|
196
|
+
## Remove the translations that have already been made
|
197
|
+
##============================================================##
|
198
|
+
data_open_ai = array.clone
|
199
|
+
data_open_ai = data_open_ai.map.with_index {|(_k, from, to), index| [index, from, to] }
|
200
|
+
data_open_ai = data_open_ai.select {|_index, from, to| !from.nil? && to.nil? }
|
201
|
+
|
202
|
+
##============================================================##
|
203
|
+
## Remove quotes surrounding the value if they are present.
|
204
|
+
## and remove to to avoid error in translation
|
205
|
+
##============================================================##
|
206
|
+
data_open_ai = data_open_ai.map do |index, from, _to|
|
207
|
+
from = from.to_s
|
208
|
+
from = from[1..-2] while (from.start_with?(DOUBLE_QUOTE) && from.end_with?(DOUBLE_QUOTE)) || (from.start_with?(SIMPLE_QUOTE) && from.end_with?(SIMPLE_QUOTE))
|
209
|
+
[index, from]
|
210
|
+
end
|
211
|
+
|
212
|
+
return array if data_open_ai.empty?
|
213
|
+
|
214
|
+
|
215
|
+
##============================================================##
|
216
|
+
## Call OpenAI API
|
217
|
+
##============================================================##
|
218
|
+
index = 0
|
219
|
+
group_size = model[:group_size]
|
220
|
+
from_iso = ISO_639.find_by_code(from).english_name.split(";").first
|
221
|
+
to_iso = ISO_639.find_by_code(to).english_name.split(";").first
|
222
|
+
ai_resuslts = []
|
223
|
+
prompt_system = "You are a translation tool from #{from_iso} to #{to_iso}\n" \
|
224
|
+
"The input is an array of pairs, where each pair contains an index and a string to translate, formatted as [index, string_to_translate]\n" \
|
225
|
+
"Your task is to create an output ARRAY where each element is a pair consisting of the index and the translated string, formatted as [index, 'string_translated']\n" \
|
226
|
+
"If a string_to_translate starts with [ and ends with ], it is considered a special string that should be treated as a JSON object. Otherwise, it's a normal string.\n" \
|
227
|
+
"\nRules to respect for JSON objects:\n" \
|
228
|
+
"- You need to translate ONLY the values of the JSON object, not the keys. Do not change anything in the format, just translate the values.\n" \
|
229
|
+
"- Respect all following rules for normal strings to translate the values\n" \
|
230
|
+
"\nRules to respect for normal strings:\n" \
|
231
|
+
"- Do not escape apostrophes in translated strings; leave them as they are.\n" \
|
232
|
+
"- Special characters, except apostrophes, that need to be escaped in translated strings should be escaped using a single backslash (\\), not double (\\\\).\n" \
|
233
|
+
"- If a string cannot be translated use the string '#{cant_be_translated}' translated as the translation value witouth quote (simple or double) quote, just the string\n" \
|
234
|
+
"- If you dont know the correct translatation use the #{cant_be_translated} strategy of the preceding point\n" \
|
235
|
+
"- Use only double quotes (\") to enclose translated strings and avoid using single quotes (').\n" \
|
236
|
+
"- Your output must ONLY be an array with the same number of pairs as the input, without any additional text or explanation. DO NOT COMMENT!\n" \
|
237
|
+
"- You need to check that the globle array is correctly closed at the end of the response. (the response must therefore end with ]] to to be consistent)"
|
238
|
+
prompt_init = "Please proceed with translating the following array:"
|
239
|
+
headers = {
|
240
|
+
"Content-Type" => "application/json",
|
241
|
+
"Authorization" => "Bearer #{ImmosquareTranslate.configuration.openai_api_key}"
|
242
|
+
}
|
243
|
+
|
244
|
+
|
245
|
+
##============================================================##
|
246
|
+
## Estimate the number of window_tokens
|
247
|
+
## https://platform.openai.com/tokenizer
|
248
|
+
## English: 75 words => 100 tokens
|
249
|
+
## French : 55 words => 100 tokens
|
250
|
+
## -----------------
|
251
|
+
## For each array value we add 5 tokens for the array format.
|
252
|
+
## [1, "my_word"],
|
253
|
+
## [ => first token
|
254
|
+
## 2 => second token
|
255
|
+
## , => third token
|
256
|
+
## " => fourth token
|
257
|
+
## ]" => fifth token
|
258
|
+
## -----------------
|
259
|
+
# data_open_ai.inspect.size => to get the total number of characters in the array
|
260
|
+
## with the array structure [""],
|
261
|
+
##============================================================##
|
262
|
+
estimation_for_100_tokens = from == "fr" ? 55 : 75
|
263
|
+
prompt_tokens_estimation = (((prompt_system.split.size + prompt_init.split.size + data_open_ai.map {|_index, from| from.split.size }.sum) / estimation_for_100_tokens * 100.0) + (data_open_ai.size * 5)).round
|
264
|
+
split_array = (prompt_tokens_estimation / model[:window_tokens].to_f).ceil
|
265
|
+
slice_size = (data_open_ai.size / split_array.to_f).round
|
266
|
+
data_open_ai_sliced = data_open_ai.each_slice(slice_size).to_a
|
267
|
+
|
268
|
+
|
269
|
+
##============================================================##
|
270
|
+
## Now each slice of the array should no be more than window_tokens
|
271
|
+
## of the model.... We can now translate each slice.
|
272
|
+
## ---------------------------------
|
273
|
+
## Normally we could send the whole slice at once and tell the api to continue if its response is not tarnished...
|
274
|
+
## But it should manage if a word is cut etc...
|
275
|
+
## For the moment we cut it into small group for which we are sure not to exceed the limit
|
276
|
+
##============================================================##
|
277
|
+
puts("fields to translate from #{from_iso} (#{from}) to #{to_iso} (#{to}) : #{data_open_ai.size}#{" by group of #{group_size}" if data_open_ai.size > group_size}")
|
278
|
+
while index < data_open_ai.size
|
279
|
+
data_group = data_open_ai[index, group_size]
|
280
|
+
|
281
|
+
|
282
|
+
begin
|
283
|
+
puts("call OPENAI Api (with model #{model[:name]}) #{" for #{data_group.size} fields (#{index}-#{index + data_group.size})" if data_open_ai.size > group_size}")
|
284
|
+
prompt = "#{prompt_init}:\n\n#{data_group.inspect}\n\n"
|
285
|
+
body = {
|
286
|
+
:model => model[:name],
|
287
|
+
:messages => [
|
288
|
+
{:role => "system", :content => prompt_system},
|
289
|
+
{:role => "user", :content => prompt}
|
290
|
+
],
|
291
|
+
:temperature => 0.0
|
292
|
+
}
|
293
|
+
t0 = Time.now
|
294
|
+
call = HTTParty.post("https://api.openai.com/v1/chat/completions", :body => body.to_json, :headers => headers, :timeout => 500)
|
295
|
+
|
296
|
+
puts("responded in #{(Time.now - t0).round(2)} seconds")
|
297
|
+
raise(call["error"]["message"]) if call.code != 200
|
298
|
+
|
299
|
+
|
300
|
+
##============================================================##
|
301
|
+
## We check that the result is complete
|
302
|
+
##============================================================##
|
303
|
+
response = JSON.parse(call.body)
|
304
|
+
choice = response["choices"][0]
|
305
|
+
raise("Result is not complete") if choice["finish_reason"] != "stop"
|
306
|
+
|
307
|
+
|
308
|
+
##============================================================##
|
309
|
+
## We calculate the estimate price of the call
|
310
|
+
##============================================================##
|
311
|
+
input_price = response["usage"]["prompt_tokens"] * (model[:input_price_for_1m] / 1_000_000)
|
312
|
+
output_price = response["usage"]["completion_tokens"] * (model[:output_price_for_1m] / 1_000_000)
|
313
|
+
price = input_price + output_price
|
314
|
+
puts("Estimate price => #{input_price.round(3)} + #{output_price.round(3)} = #{price.round(3)} USD")
|
315
|
+
|
316
|
+
##============================================================##
|
317
|
+
## We check that the result is an array
|
318
|
+
##============================================================##
|
319
|
+
content = eval(choice["message"]["content"])
|
320
|
+
raise("Is not an array") if !content.is_a?(Array)
|
321
|
+
|
322
|
+
##============================================================##
|
323
|
+
## We save the result
|
324
|
+
##============================================================##
|
325
|
+
content.each do |index, translation|
|
326
|
+
ai_resuslts << [index, translation == cant_be_translated ? nil : translation]
|
327
|
+
end
|
328
|
+
rescue StandardError => e
|
329
|
+
puts("error OPEN AI API => #{e.message}")
|
330
|
+
puts(e.message)
|
331
|
+
puts(e.backtrace)
|
332
|
+
end
|
333
|
+
index += group_size
|
334
|
+
end
|
335
|
+
|
336
|
+
|
337
|
+
##============================================================##
|
338
|
+
## We put the translations in the original array
|
339
|
+
##============================================================##
|
340
|
+
ai_resuslts.each do |index, translation|
|
341
|
+
begin
|
342
|
+
array[index.to_i][2] = translation
|
343
|
+
rescue StandardError => e
|
344
|
+
puts(e.message)
|
345
|
+
end
|
346
|
+
end
|
347
|
+
|
348
|
+
##============================================================##
|
349
|
+
## We return the modified array
|
350
|
+
##============================================================##
|
351
|
+
array.map.with_index do |(k, from, to), index|
|
352
|
+
from = from.to_s
|
353
|
+
to = "#{DOUBLE_QUOTE}#{to}#{DOUBLE_QUOTE}" if ai_resuslts.find {|i, _t| i == index } && ((from.start_with?(DOUBLE_QUOTE) && from.end_with?(DOUBLE_QUOTE)) || (from.start_with?(SIMPLE_QUOTE) && from.end_with?(SIMPLE_QUOTE)))
|
354
|
+
[k, from, to]
|
355
|
+
end
|
356
|
+
end
|
357
|
+
|
358
|
+
|
359
|
+
end
|
360
|
+
end
|
361
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require_relative "immosquare-translate/configuration"
|
2
|
+
require_relative "immosquare-translate/shared_methods"
|
3
|
+
require_relative "immosquare-translate/yml_translator"
|
4
|
+
require_relative "immosquare-translate/translator"
|
5
|
+
|
6
|
+
|
7
|
+
|
8
|
+
##===========================================================================##
|
9
|
+
##
|
10
|
+
##===========================================================================##
|
11
|
+
module ImmosquareTranslate
|
12
|
+
class << self
|
13
|
+
|
14
|
+
##===========================================================================##
|
15
|
+
## Gem configuration
|
16
|
+
##===========================================================================##
|
17
|
+
attr_writer :configuration
|
18
|
+
|
19
|
+
def configuration
|
20
|
+
@configuration ||= Configuration.new
|
21
|
+
end
|
22
|
+
|
23
|
+
def config
|
24
|
+
yield(configuration)
|
25
|
+
end
|
26
|
+
|
27
|
+
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
namespace :immosquare_yaml do
|
2
|
+
##============================================================##
|
3
|
+
## Function to translate translation files in rails app
|
4
|
+
## rake immosquare_yaml:translate SOURCE_LOCALE=fr
|
5
|
+
##============================================================##
|
6
|
+
desc "Translate translation files in rails app"
|
7
|
+
task :translate => :environment do
|
8
|
+
begin
|
9
|
+
source_locale = ENV.fetch("SOURCE_LOCALE", nil) || "fr"
|
10
|
+
reset_translations = ENV.fetch("RESET_TRANSLATIONS", nil) || false
|
11
|
+
reset_translations = reset_translations == "true"
|
12
|
+
|
13
|
+
raise("Please provide a valid locale") if !I18n.available_locales.map(&:to_s).include?(source_locale)
|
14
|
+
raise("Please provide a valid boolean for reset_translations") if ![true, false].include?(reset_translations)
|
15
|
+
|
16
|
+
locales = I18n.available_locales.map(&:to_s).reject {|l| l == source_locale }
|
17
|
+
puts("Translating from #{source_locale} to #{locales.join(", ")} with reset_translations=#{reset_translations}")
|
18
|
+
Dir.glob("#{Rails.root}/config/locales/**/*#{source_locale}.yml").each do |file|
|
19
|
+
locales.each do |locale|
|
20
|
+
ImmosquareYaml::Translate.translate(file, locale, :reset_translations => reset_translations)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
rescue StandardError => e
|
24
|
+
puts(e.message)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
##============================================================##
|
29
|
+
## Function to clean translation files in rails app
|
30
|
+
##============================================================##
|
31
|
+
desc "Clean translation files in rails app"
|
32
|
+
task :clean => :environment do
|
33
|
+
Dir.glob("#{Rails.root}/config/locales/**/*.yml").each do |file|
|
34
|
+
ImmosquareYaml.clean(file)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
metadata
ADDED
@@ -0,0 +1,103 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: immosquare-translate
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- IMMO SQUARE
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2024-03-18 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: httparty
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: immosquare-yaml
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
- - ">="
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: 0.1.26
|
37
|
+
type: :runtime
|
38
|
+
prerelease: false
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - "~>"
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 0.1.26
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: iso-639
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '0'
|
61
|
+
description: ImmosquareTranslate brings the power of OpenAI to Ruby applications,
|
62
|
+
offering the ability to translate not just YAML files, but also arrays, web pages,
|
63
|
+
and other data structures. Tailored for developers in multilingual settings, it
|
64
|
+
streamlines the translation workflow, ensuring accurate, context-aware translations
|
65
|
+
across different content types.
|
66
|
+
email:
|
67
|
+
- jules@immosquare.com
|
68
|
+
executables: []
|
69
|
+
extensions: []
|
70
|
+
extra_rdoc_files: []
|
71
|
+
files:
|
72
|
+
- lib/immosquare-translate.rb
|
73
|
+
- lib/immosquare-translate/configuration.rb
|
74
|
+
- lib/immosquare-translate/shared_methods.rb
|
75
|
+
- lib/immosquare-translate/translator.rb
|
76
|
+
- lib/immosquare-translate/version.rb
|
77
|
+
- lib/immosquare-translate/yml_translator.rb
|
78
|
+
- lib/tasks/immosquare-yaml.rake
|
79
|
+
homepage: https://github.com/IMMOSQUARE/immosquare-translate
|
80
|
+
licenses:
|
81
|
+
- MIT
|
82
|
+
metadata: {}
|
83
|
+
post_install_message:
|
84
|
+
rdoc_options: []
|
85
|
+
require_paths:
|
86
|
+
- lib
|
87
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
88
|
+
requirements:
|
89
|
+
- - ">="
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: 2.7.2
|
92
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
requirements: []
|
98
|
+
rubygems_version: 3.4.13
|
99
|
+
signing_key:
|
100
|
+
specification_version: 4
|
101
|
+
summary: AI-powered translations for Ruby applications, supporting a wide range of
|
102
|
+
formats.
|
103
|
+
test_files: []
|