translatomatic 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. data/.translatomatic/config.yml +18 -0
  3. data/.travis.yml +33 -33
  4. data/Gemfile +6 -4
  5. data/README.de.md +53 -18
  6. data/README.es.md +55 -20
  7. data/README.fr.md +54 -19
  8. data/README.it.md +58 -23
  9. data/README.ja.md +54 -19
  10. data/README.ko.md +58 -23
  11. data/README.md +167 -141
  12. data/README.ms.md +51 -16
  13. data/README.pt.md +58 -23
  14. data/README.ru.md +53 -18
  15. data/README.sv.md +53 -18
  16. data/README.zh.md +53 -18
  17. data/bin/translatomatic +6 -6
  18. data/bin/travis +24 -26
  19. data/config/locales/translatomatic/de.yml +22 -11
  20. data/config/locales/translatomatic/en.yml +21 -12
  21. data/config/locales/translatomatic/es.yml +22 -11
  22. data/config/locales/translatomatic/fr.yml +22 -12
  23. data/config/locales/translatomatic/it.yml +22 -11
  24. data/config/locales/translatomatic/ja.yml +22 -11
  25. data/config/locales/translatomatic/ko.yml +22 -11
  26. data/config/locales/translatomatic/ms.yml +22 -11
  27. data/config/locales/translatomatic/pt.yml +22 -11
  28. data/config/locales/translatomatic/ru.yml +22 -11
  29. data/config/locales/translatomatic/sv.yml +22 -11
  30. data/config/locales/translatomatic/zh.yml +22 -11
  31. data/db/migrate/201712170000_initial.rb +25 -25
  32. data/lib/translatomatic/cli/base.rb +81 -73
  33. data/lib/translatomatic/cli/config.rb +110 -81
  34. data/lib/translatomatic/cli/main.rb +85 -72
  35. data/lib/translatomatic/cli/translate.rb +141 -106
  36. data/lib/translatomatic/cli.rb +8 -8
  37. data/lib/translatomatic/config.rb +302 -155
  38. data/lib/translatomatic/converter.rb +28 -260
  39. data/lib/translatomatic/database.rb +134 -134
  40. data/lib/translatomatic/define_options.rb +22 -0
  41. data/lib/translatomatic/escaped_unicode.rb +0 -0
  42. data/lib/translatomatic/extractor/base.rb +16 -16
  43. data/lib/translatomatic/extractor/ruby.rb +6 -6
  44. data/lib/translatomatic/extractor.rb +5 -5
  45. data/lib/translatomatic/file_translator.rb +269 -0
  46. data/lib/translatomatic/http_request.rb +162 -162
  47. data/lib/translatomatic/locale.rb +76 -76
  48. data/lib/translatomatic/logger.rb +23 -23
  49. data/lib/translatomatic/model/locale.rb +25 -25
  50. data/lib/translatomatic/model/text.rb +19 -19
  51. data/lib/translatomatic/model.rb +1 -1
  52. data/lib/translatomatic/option.rb +37 -41
  53. data/lib/translatomatic/progress_updater.rb +13 -13
  54. data/lib/translatomatic/resource_file/base.rb +269 -192
  55. data/lib/translatomatic/resource_file/csv.rb +37 -0
  56. data/lib/translatomatic/resource_file/html.rb +54 -47
  57. data/lib/translatomatic/resource_file/markdown.rb +50 -55
  58. data/lib/translatomatic/resource_file/plist.rb +153 -19
  59. data/lib/translatomatic/resource_file/po.rb +107 -0
  60. data/lib/translatomatic/resource_file/properties.rb +91 -90
  61. data/lib/translatomatic/resource_file/resw.rb +50 -30
  62. data/lib/translatomatic/resource_file/subtitle.rb +75 -0
  63. data/lib/translatomatic/resource_file/text.rb +24 -30
  64. data/lib/translatomatic/resource_file/xcode_strings.rb +75 -80
  65. data/lib/translatomatic/resource_file/xml.rb +98 -91
  66. data/lib/translatomatic/resource_file/yaml.rb +94 -116
  67. data/lib/translatomatic/resource_file.rb +87 -78
  68. data/lib/translatomatic/string.rb +188 -188
  69. data/lib/translatomatic/tmx/document.rb +99 -99
  70. data/lib/translatomatic/translation_result.rb +63 -63
  71. data/lib/translatomatic/{converter_stats.rb → translation_stats.rb} +17 -17
  72. data/lib/translatomatic/translator/base.rb +1 -1
  73. data/lib/translatomatic/translator/google.rb +2 -0
  74. data/lib/translatomatic/translator.rb +10 -2
  75. data/lib/translatomatic/util.rb +45 -45
  76. data/lib/translatomatic/version.rb +7 -7
  77. data/lib/translatomatic.rb +52 -49
  78. data/translatomatic.gemspec +3 -2
  79. metadata +25 -5
@@ -0,0 +1,269 @@
1
+ # The file translator ties together functionality from translators,
2
+ # resource files, and the database to convert files from one
3
+ # language to another.
4
+ class Translatomatic::FileTranslator
5
+
6
+ # @return [Array<Translatomatic::Model::Text>] A list of translations saved to the database
7
+ attr_reader :db_translations
8
+
9
+ # Create a converter to translate files
10
+ #
11
+ # @param options [Hash<Symbol,Object>] converter and/or translator options.
12
+ def initialize(options = {})
13
+ @dry_run = options[:dry_run]
14
+ @listener = options[:listener]
15
+ @translators = Translatomatic::Translator.resolve(options[:translator], options)
16
+ raise t("file_translator.translator_required") if @translators.empty?
17
+ @translators.each { |i| i.listener = @listener } if @listener
18
+
19
+ # use database by default if we're connected to a database
20
+ use_db = options.include?(:use_database) ? options[:use_database] : true
21
+ @use_db = use_db && ActiveRecord::Base.connected?
22
+ log.debug(t("file_translator.database_disabled")) unless @use_db
23
+
24
+ @db_translations = []
25
+ @translations = {} # map of original text to Translation
26
+ end
27
+
28
+ # @return [Translatomatic::TranslationStats] Translation statistics
29
+ def stats
30
+ @stats ||= Translatomatic::TranslationStats.new(@translations)
31
+ end
32
+
33
+ # Translate properties of source_file to the target locale.
34
+ # Does not write changes to disk.
35
+ #
36
+ # @param file [String, Translatomatic::ResourceFile] File to translate
37
+ # @param to_locale [String] The target locale, e.g. "fr"
38
+ # @return [Translatomatic::ResourceFile] The translated resource file
39
+ def translate(file, to_locale)
40
+ file = resource_file(file)
41
+ to_locale = parse_locale(to_locale)
42
+
43
+ # do nothing if target language is the same as source language
44
+ return file if file.locale.language == to_locale.language
45
+
46
+ result = Translatomatic::TranslationResult.new(file, to_locale)
47
+
48
+ # translate using strings from the database first
49
+ each_translator(result) { translate_properties_with_db(result) }
50
+ # send remaining unknown strings to translator
51
+ each_translator(result) { translate_properties_with_translator(result) }
52
+
53
+ log.debug(t("file_translator.stats", from_db: stats.from_db,
54
+ from_translator: stats.from_translator,
55
+ untranslated: result.untranslated.length))
56
+ @listener.untranslated_texts(result.untranslated) if @listener
57
+
58
+ file.properties = result.properties
59
+ file.locale = to_locale
60
+ file
61
+ end
62
+
63
+ # Translates a resource file and writes results to a target resource file.
64
+ # The path of the target resource file is automatically determined.
65
+ #
66
+ # @param source [Translatomatic::ResourceFile] The source
67
+ # @param to_locale [String] The target locale, e.g. "fr"
68
+ # @return [Translatomatic::ResourceFile] The translated resource file
69
+ def translate_to_file(source, to_locale)
70
+ # Automatically determines the target filename based on target locale.
71
+ source = resource_file(source)
72
+ target = Translatomatic::ResourceFile.load(source.path)
73
+ target.path = source.locale_path(to_locale)
74
+
75
+ log.info(t("file_translator.translating", source: source,
76
+ source_locale: source.locale, target: target, target_locale: to_locale))
77
+ translate(target, to_locale)
78
+ unless @dry_run
79
+ target.path.parent.mkpath
80
+ target.save
81
+ end
82
+ target
83
+ end
84
+
85
+ private
86
+
87
+ include Translatomatic::Util
88
+ include Translatomatic::DefineOptions
89
+
90
+ define_options(
91
+ { name: :dry_run, type: :boolean, aliases: "-n",
92
+ desc: t("file_translator.dry_run"),
93
+ command_line_only: true
94
+ },
95
+ { name: :use_database, type: :boolean, default: true,
96
+ desc: t("file_translator.use_database")
97
+ }
98
+ )
99
+
100
+ def each_translator(result)
101
+ @translators.each do |translator|
102
+ break if result.untranslated.empty?
103
+ @current_translator = translator
104
+ yield
105
+ end
106
+ end
107
+
108
+ # Attempt to restore interpolated variable names in the translation.
109
+ # If variable names cannot be restored, sets the translation result to nil.
110
+ # @param result [Translatomatic::TranslationResult] translation result
111
+ # @param translation [Translatomatic::Translation] translation
112
+ # @return [void]
113
+ def restore_variables(result, translation)
114
+ file = result.file
115
+ return unless file.class.supports_variable_interpolation?
116
+
117
+ # find variables in the original string
118
+ variables = string_variables(translation.original, file.locale, file)
119
+ # find variables in the translated string
120
+ translated_variables = string_variables(translation.result, result.to_locale, file)
121
+
122
+ if variables.length == translated_variables.length
123
+ # we can restore variables. sort by largest offset first.
124
+ # not using translation() method as that adds to @translations hash.
125
+ conversions = variables.zip(translated_variables).collect {
126
+ |v1, v2| Translatomatic::Translation.new(v1, v2)
127
+ }
128
+ conversions.sort_by! { |t| -t.original.offset }
129
+ conversions.each do |conversion|
130
+ v1 = conversion.original
131
+ v2 = conversion.result
132
+ translation.result[v2.offset, v2.length] = v1.value
133
+ end
134
+ else
135
+ # unable to restore interpolated variable names
136
+ log.debug("#{@current_translator.name}: unable to restore variables: #{translation.result}")
137
+ translation.result = nil # mark result as invalid
138
+ end
139
+ end
140
+
141
+ def string_variables(value, locale, file)
142
+ string(value, locale).substrings(file.variable_regex)
143
+ end
144
+
145
+ def resource_file(path)
146
+ if path.kind_of?(Translatomatic::ResourceFile::Base)
147
+ path
148
+ else
149
+ file = Translatomatic::ResourceFile.load(path)
150
+ raise t("file.unsupported", file: path) unless file
151
+ file
152
+ end
153
+ end
154
+
155
+ # update result with translations from the database.
156
+ def translate_properties_with_db(result)
157
+ db_texts = []
158
+ unless database_disabled?
159
+ translations = []
160
+ untranslated = hashify(result.untranslated)
161
+ db_texts = find_database_translations(result, result.untranslated.to_a)
162
+ db_texts.each do |db_text|
163
+ from_text = db_text.from_text.value
164
+ if untranslated[from_text]
165
+ translation = translation(untranslated[from_text], db_text.value, true)
166
+ restore_variables(result, translation)
167
+ translations << translation
168
+ end
169
+ end
170
+
171
+ result.update_strings(translations)
172
+ @listener.translated_texts(db_texts) if @listener
173
+ end
174
+ db_texts
175
+ end
176
+
177
+ # update result with translations from the translator.
178
+ def translate_properties_with_translator(result)
179
+ untranslated = result.untranslated.to_a.select { |i| translatable?(i) }
180
+ translated = []
181
+ if !untranslated.empty? && !@dry_run
182
+ untranslated_strings = untranslated.collect { |i| i.to_s }
183
+ log.debug("translating: #{untranslated_strings}")
184
+ translated = @current_translator.translate(untranslated_strings,
185
+ result.from_locale, result.to_locale
186
+ )
187
+
188
+ # create list of translations, filtering out invalid translations
189
+ translations = []
190
+ untranslated.zip(translated).each do |from, to|
191
+ translation = translation(from, to, false)
192
+ restore_variables(result, translation)
193
+ translations << translation
194
+ end
195
+
196
+ result.update_strings(translations)
197
+ unless database_disabled?
198
+ save_database_translations(result, translations)
199
+ end
200
+ end
201
+ translated
202
+ end
203
+
204
+ def translation(from, to, from_database = false)
205
+ translator = @current_translator.name
206
+ t = Translatomatic::Translation.new(from, to, translator, from_database)
207
+ @translations[from] = t
208
+ t
209
+ end
210
+
211
+ def database_disabled?
212
+ !@use_db
213
+ end
214
+
215
+ def parse_locale(locale)
216
+ Translatomatic::Locale.parse(locale)
217
+ end
218
+
219
+ def translatable?(string)
220
+ # don't translate numbers
221
+ string && !string.match(/\A\s*\z/) && !string.match(/\A[\d,]+\z/)
222
+ end
223
+
224
+ def save_database_translations(result, translations)
225
+ ActiveRecord::Base.transaction do
226
+ from = db_locale(result.from_locale)
227
+ to = db_locale(result.to_locale)
228
+ translations.each do |translation|
229
+ next if translation.result.nil? # skip invalid translations
230
+ save_database_translation(from, to, translation)
231
+ end
232
+ end
233
+ end
234
+
235
+ def save_database_translation(from_locale, to_locale, translation)
236
+ original_text = Translatomatic::Model::Text.find_or_create_by!(
237
+ locale: from_locale,
238
+ value: translation.original.to_s
239
+ )
240
+
241
+ text = Translatomatic::Model::Text.find_or_create_by!(
242
+ locale: to_locale,
243
+ value: translation.result.to_s,
244
+ from_text: original_text,
245
+ translator: @current_translator.name
246
+ )
247
+ @db_translations += [original_text, text]
248
+ text
249
+ end
250
+
251
+ def find_database_translations(result, untranslated)
252
+ from = db_locale(result.from_locale)
253
+ to = db_locale(result.to_locale)
254
+
255
+ Translatomatic::Model::Text.where({
256
+ locale: to,
257
+ translator: @current_translator.name,
258
+ from_texts_texts: {
259
+ locale_id: from,
260
+ # convert untranslated set to strings
261
+ value: untranslated.collect { |i| i.to_s }
262
+ }
263
+ }).joins(:from_text)
264
+ end
265
+
266
+ def db_locale(locale)
267
+ Translatomatic::Model::Locale.from_tag(locale)
268
+ end
269
+ end
@@ -1,162 +1,162 @@
1
- require 'securerandom'
2
- require 'net/http'
3
-
4
- module Translatomatic
5
- # HTTP request
6
- # wrapper for Net::HTTP functionality
7
- class HTTPRequest
8
-
9
- # @return [String] the text to use to denote multipart boundaries. By
10
- # default, a random hexadecimal string is used.
11
- attr_accessor :multipart_boundary
12
-
13
- # @param url [String,URI] URL of the request
14
- # @return [Translatomatic::HTTPRequest] Create a new request
15
- def initialize(url)
16
- @uri = url.respond_to?(:host) ? url : URI.parse(url)
17
- @multipart_boundary = SecureRandom.hex(16)
18
- end
19
-
20
- # Start the HTTP request. Yields a http object.
21
- # @param options [Hash<Symbol,Object>] Request options
22
- # @return [Object] Result of the block
23
- def start(options = {})
24
- options = options.merge(use_ssl: @uri.scheme == "https")
25
- result = nil
26
- Net::HTTP.start(@uri.host, @uri.port, options) do |http|
27
- @http = http
28
- result = yield http
29
- end
30
- @http = nil
31
- result
32
- end
33
-
34
- # Send a HTTP GET request
35
- # @param query [Hash<String,String>] Optional query parameters
36
- # @return [Net::HTTP::Response]
37
- def get(query = nil)
38
- uri = @uri
39
- if query
40
- uri = @uri.dup
41
- uri.query = URI.encode_www_form(query)
42
- end
43
- request = Net::HTTP::Get.new(uri)
44
- request['User-Agent'] = USER_AGENT
45
- send_request(request)
46
- end
47
-
48
- # Send an HTTP POST request
49
- # @param body [String,Hash] Body of the request
50
- # @return [Net::HTTP::Response]
51
- def post(body, options = {})
52
- request = Net::HTTP::Post.new(@uri)
53
- request['User-Agent'] = USER_AGENT
54
- content_type = options[:content_type]
55
-
56
- if options[:multipart]
57
- content_type = "multipart/form-data; boundary=#{@multipart_boundary}"
58
- request.body = multipartify(body)
59
- elsif body.kind_of?(Hash)
60
- # set_form_data does url encoding
61
- request.set_form_data(body)
62
- else
63
- request.body = body
64
- end
65
- request.content_type = content_type if content_type
66
-
67
- send_request(request)
68
- end
69
-
70
- # Create a file parameter for a multipart POST request
71
- # @return [FileParam] A new file parameter
72
- def file(*args)
73
- FileParam.new(*args)
74
- end
75
-
76
- # Create a parameter for a multipart POST request
77
- # @return [Param] A new parameter
78
- def param(*args)
79
- Param.new(*args)
80
- end
81
-
82
- private
83
-
84
- USER_AGENT = "Translatomatic #{VERSION} (+#{URL})"
85
-
86
- # Formats a basic string key/value pair for a multipart post
87
- class Param
88
- attr_accessor :key, :value
89
-
90
- # @return [String] Representation of this parameter as it appears
91
- # within a multipart post request.
92
- def to_s
93
- return header(header_data) + "\r\n#{value}\r\n"
94
- end
95
-
96
- private
97
-
98
- def initialize(key:, value:)
99
- @key = key
100
- @value = value
101
- end
102
-
103
- def header_data
104
- name = CGI::escape(key.to_s)
105
- { "Content-Disposition": "form-data", name: %Q("#{name}") }
106
- end
107
-
108
- def header(options)
109
- out = []
110
- idx = 0
111
- options.each do |key, value|
112
- separator = idx == 0 ? ": " : "="
113
- out << "#{key}#{separator}#{value}"
114
- idx += 1
115
- end
116
- out.join("; ") + "\r\n"
117
- end
118
- end
119
-
120
- # Formats the contents of a file or string for a multipart post
121
- class FileParam < Param
122
- attr_accessor :filename, :content, :mime_type
123
-
124
- # (see Param#to_s)
125
- def to_s
126
- return header(header_data) +
127
- header("Content-Type": mime_type) + "\r\n#{content}\r\n"
128
- end
129
-
130
- private
131
-
132
- def initialize(key:, filename:, content:, mime_type:)
133
- @key = key
134
- @filename = filename
135
- @content = content
136
- @mime_type = mime_type
137
- end
138
-
139
- def header_data
140
- super.merge({ filename: %Q("#{filename}") })
141
- end
142
- end
143
-
144
- def multipartify(parts)
145
- string_parts = parts.collect do |p|
146
- "--" + @multipart_boundary + "\r\n" + p.to_s
147
- end
148
- string_parts.join("") + "--" + @multipart_boundary + "--\r\n"
149
- end
150
-
151
- def send_request(req)
152
- if @http
153
- response = @http.request(req)
154
- else
155
- response = start { |http| http.request(req) }
156
- end
157
- raise response.body unless response.kind_of? Net::HTTPSuccess
158
- response
159
- end
160
-
161
- end # class
162
- end # module
1
+ require 'securerandom'
2
+ require 'net/http'
3
+
4
+ module Translatomatic
5
+ # HTTP request
6
+ # wrapper for Net::HTTP functionality
7
+ class HTTPRequest
8
+
9
+ # @return [String] the text to use to denote multipart boundaries. By
10
+ # default, a random hexadecimal string is used.
11
+ attr_accessor :multipart_boundary
12
+
13
+ # @param url [String,URI] URL of the request
14
+ # @return [Translatomatic::HTTPRequest] Create a new request
15
+ def initialize(url)
16
+ @uri = url.respond_to?(:host) ? url : URI.parse(url)
17
+ @multipart_boundary = SecureRandom.hex(16)
18
+ end
19
+
20
+ # Start the HTTP request. Yields a http object.
21
+ # @param options [Hash<Symbol,Object>] Request options
22
+ # @return [Object] Result of the block
23
+ def start(options = {})
24
+ options = options.merge(use_ssl: @uri.scheme == "https")
25
+ result = nil
26
+ Net::HTTP.start(@uri.host, @uri.port, options) do |http|
27
+ @http = http
28
+ result = yield http
29
+ end
30
+ @http = nil
31
+ result
32
+ end
33
+
34
+ # Send a HTTP GET request
35
+ # @param query [Hash<String,String>] Optional query parameters
36
+ # @return [Net::HTTP::Response]
37
+ def get(query = nil)
38
+ uri = @uri
39
+ if query
40
+ uri = @uri.dup
41
+ uri.query = URI.encode_www_form(query)
42
+ end
43
+ request = Net::HTTP::Get.new(uri)
44
+ request['User-Agent'] = USER_AGENT
45
+ send_request(request)
46
+ end
47
+
48
+ # Send an HTTP POST request
49
+ # @param body [String,Hash] Body of the request
50
+ # @return [Net::HTTP::Response]
51
+ def post(body, options = {})
52
+ request = Net::HTTP::Post.new(@uri)
53
+ request['User-Agent'] = USER_AGENT
54
+ content_type = options[:content_type]
55
+
56
+ if options[:multipart]
57
+ content_type = "multipart/form-data; boundary=#{@multipart_boundary}"
58
+ request.body = multipartify(body)
59
+ elsif body.kind_of?(Hash)
60
+ # set_form_data does url encoding
61
+ request.set_form_data(body)
62
+ else
63
+ request.body = body
64
+ end
65
+ request.content_type = content_type if content_type
66
+
67
+ send_request(request)
68
+ end
69
+
70
+ # Create a file parameter for a multipart POST request
71
+ # @return [FileParam] A new file parameter
72
+ def file(*args)
73
+ FileParam.new(*args)
74
+ end
75
+
76
+ # Create a parameter for a multipart POST request
77
+ # @return [Param] A new parameter
78
+ def param(*args)
79
+ Param.new(*args)
80
+ end
81
+
82
+ private
83
+
84
+ USER_AGENT = "Translatomatic #{VERSION} (+#{URL})"
85
+
86
+ # Formats a basic string key/value pair for a multipart post
87
+ class Param
88
+ attr_accessor :key, :value
89
+
90
+ # @return [String] Representation of this parameter as it appears
91
+ # within a multipart post request.
92
+ def to_s
93
+ return header(header_data) + "\r\n#{value}\r\n"
94
+ end
95
+
96
+ private
97
+
98
+ def initialize(key:, value:)
99
+ @key = key
100
+ @value = value
101
+ end
102
+
103
+ def header_data
104
+ name = CGI::escape(key.to_s)
105
+ { "Content-Disposition": "form-data", name: %Q("#{name}") }
106
+ end
107
+
108
+ def header(options)
109
+ out = []
110
+ idx = 0
111
+ options.each do |key, value|
112
+ separator = idx == 0 ? ": " : "="
113
+ out << "#{key}#{separator}#{value}"
114
+ idx += 1
115
+ end
116
+ out.join("; ") + "\r\n"
117
+ end
118
+ end
119
+
120
+ # Formats the contents of a file or string for a multipart post
121
+ class FileParam < Param
122
+ attr_accessor :filename, :content, :mime_type
123
+
124
+ # (see Param#to_s)
125
+ def to_s
126
+ return header(header_data) +
127
+ header("Content-Type": mime_type) + "\r\n#{content}\r\n"
128
+ end
129
+
130
+ private
131
+
132
+ def initialize(key:, filename:, content:, mime_type:)
133
+ @key = key
134
+ @filename = filename
135
+ @content = content
136
+ @mime_type = mime_type
137
+ end
138
+
139
+ def header_data
140
+ super.merge({ filename: %Q("#{filename}") })
141
+ end
142
+ end
143
+
144
+ def multipartify(parts)
145
+ string_parts = parts.collect do |p|
146
+ "--" + @multipart_boundary + "\r\n" + p.to_s
147
+ end
148
+ string_parts.join("") + "--" + @multipart_boundary + "--\r\n"
149
+ end
150
+
151
+ def send_request(req)
152
+ if @http
153
+ response = @http.request(req)
154
+ else
155
+ response = start { |http| http.request(req) }
156
+ end
157
+ raise response.body unless response.kind_of? Net::HTTPSuccess
158
+ response
159
+ end
160
+
161
+ end # class
162
+ end # module