yury-twine 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +2 -0
  3. data/LICENSE +30 -0
  4. data/README.md +230 -0
  5. data/bin/twine +7 -0
  6. data/lib/twine.rb +36 -0
  7. data/lib/twine/cli.rb +200 -0
  8. data/lib/twine/encoding.rb +22 -0
  9. data/lib/twine/formatters.rb +20 -0
  10. data/lib/twine/formatters/abstract.rb +187 -0
  11. data/lib/twine/formatters/android.rb +254 -0
  12. data/lib/twine/formatters/apple.rb +328 -0
  13. data/lib/twine/output_processor.rb +57 -0
  14. data/lib/twine/placeholders.rb +54 -0
  15. data/lib/twine/plugin.rb +62 -0
  16. data/lib/twine/runner.rb +332 -0
  17. data/lib/twine/twine_file.rb +266 -0
  18. data/lib/twine/version.rb +3 -0
  19. data/test/command_test.rb +14 -0
  20. data/test/fixtures/consume_loc_drop.zip +0 -0
  21. data/test/fixtures/enc_utf16be.dummy +0 -0
  22. data/test/fixtures/enc_utf16be_bom.dummy +0 -0
  23. data/test/fixtures/enc_utf16le.dummy +0 -0
  24. data/test/fixtures/enc_utf16le_bom.dummy +0 -0
  25. data/test/fixtures/enc_utf8.dummy +2 -0
  26. data/test/fixtures/formatter_android.xml +15 -0
  27. data/test/fixtures/formatter_apple.strings +20 -0
  28. data/test/fixtures/formatter_django.po +30 -0
  29. data/test/fixtures/formatter_flash.properties +15 -0
  30. data/test/fixtures/formatter_gettext.po +26 -0
  31. data/test/fixtures/formatter_jquery.json +7 -0
  32. data/test/fixtures/formatter_tizen.xml +15 -0
  33. data/test/fixtures/gettext_multiline.po +10 -0
  34. data/test/fixtures/twine_accent_values.txt +13 -0
  35. data/test/test_abstract_formatter.rb +165 -0
  36. data/test/test_cli.rb +304 -0
  37. data/test/test_consume_loc_drop.rb +27 -0
  38. data/test/test_consume_localization_file.rb +119 -0
  39. data/test/test_formatters.rb +363 -0
  40. data/test/test_generate_all_localization_files.rb +102 -0
  41. data/test/test_generate_loc_drop.rb +80 -0
  42. data/test/test_generate_localization_file.rb +91 -0
  43. data/test/test_output_processor.rb +85 -0
  44. data/test/test_placeholders.rb +84 -0
  45. data/test/test_twine_definition.rb +111 -0
  46. data/test/test_twine_file.rb +58 -0
  47. data/test/test_validate_twine_file.rb +61 -0
  48. data/test/twine_file_dsl.rb +46 -0
  49. data/test/twine_test.rb +48 -0
  50. metadata +179 -0
@@ -0,0 +1,332 @@
1
+ require 'tmpdir'
2
+ require 'fileutils'
3
+
4
+ Twine::Plugin.new # Initialize plugins first in Runner.
5
+
6
+ module Twine
7
+ class Runner
8
+ def self.run(args)
9
+ options = CLI.parse(args)
10
+
11
+ twine_file = TwineFile.new
12
+ twine_file.read options[:twine_file]
13
+ runner = new(options, twine_file)
14
+
15
+ case options[:command]
16
+ when 'generate-localization-file'
17
+ runner.generate_localization_file
18
+ when 'generate-all-localization-files'
19
+ runner.generate_all_localization_files
20
+ when 'consume-localization-file'
21
+ runner.consume_localization_file
22
+ when 'consume-all-localization-files'
23
+ runner.consume_all_localization_files
24
+ when 'generate-loc-drop'
25
+ runner.generate_loc_drop
26
+ when 'consume-loc-drop'
27
+ runner.consume_loc_drop
28
+ when 'validate-twine-file'
29
+ runner.validate_twine_file
30
+ end
31
+ end
32
+
33
+ def initialize(options = {}, twine_file = TwineFile.new)
34
+ @options = options
35
+ @twine_file = twine_file
36
+ end
37
+
38
+ def write_twine_data(path)
39
+ if @options[:developer_language]
40
+ @twine_file.set_developer_language_code(@options[:developer_language])
41
+ end
42
+ @twine_file.write(path)
43
+ end
44
+
45
+ def generate_localization_file
46
+ validate_twine_file if @options[:validate]
47
+
48
+ lang = nil
49
+ lang = @options[:languages][0] if @options[:languages]
50
+
51
+ formatter, lang = prepare_read_write(@options[:output_path], lang)
52
+ output = formatter.format_file(lang)
53
+
54
+ raise Twine::Error.new "Nothing to generate! The resulting file would not contain any translations." unless output
55
+
56
+ IO.write(@options[:output_path], output, encoding: encoding)
57
+ end
58
+
59
+ def generate_all_localization_files
60
+ validate_twine_file if @options[:validate]
61
+
62
+ if !File.directory?(@options[:output_path])
63
+ if @options[:create_folders]
64
+ FileUtils.mkdir_p(@options[:output_path])
65
+ else
66
+ raise Twine::Error.new("Directory does not exist: #{@options[:output_path]}")
67
+ end
68
+ end
69
+
70
+ formatter_for_directory = find_formatter { |f| f.can_handle_directory?(@options[:output_path]) }
71
+ formatter = formatter_for_format(@options[:format]) || formatter_for_directory
72
+
73
+ unless formatter
74
+ raise Twine::Error.new "Could not determine format given the contents of #{@options[:output_path]}"
75
+ end
76
+
77
+ file_name = @options[:file_name] || formatter.default_file_name
78
+ if @options[:create_folders]
79
+ @twine_file.language_codes.each do |lang|
80
+ output_path = File.join(@options[:output_path], formatter.output_path_for_language(lang))
81
+
82
+ FileUtils.mkdir_p(output_path)
83
+
84
+ file_path = File.join(output_path, file_name)
85
+
86
+ output = formatter.format_file(lang)
87
+ unless output
88
+ Twine::stderr.puts "Skipping file at path #{file_path} since it would not contain any translations."
89
+ next
90
+ end
91
+
92
+ IO.write(file_path, output, encoding: encoding)
93
+ end
94
+ else
95
+ language_found = false
96
+ Dir.foreach(@options[:output_path]) do |item|
97
+ next if item == "." or item == ".."
98
+
99
+ output_path = File.join(@options[:output_path], item)
100
+ next unless File.directory?(output_path)
101
+
102
+ lang = formatter.determine_language_given_path(output_path)
103
+ next unless lang
104
+
105
+ language_found = true
106
+
107
+ file_path = File.join(output_path, file_name)
108
+ output = formatter.format_file(lang)
109
+ unless output
110
+ Twine::stderr.puts "Skipping file at path #{file_path} since it would not contain any translations."
111
+ next
112
+ end
113
+
114
+ IO.write(file_path, output, encoding: encoding)
115
+ end
116
+
117
+ unless language_found
118
+ raise Twine::Error.new("Failed to generate any files: No languages found at #{@options[:output_path]}")
119
+ end
120
+ end
121
+
122
+ end
123
+
124
+ def consume_localization_file
125
+ lang = nil
126
+ if @options[:languages]
127
+ lang = @options[:languages][0]
128
+ end
129
+
130
+ read_localization_file(@options[:input_path], lang)
131
+ output_path = @options[:output_path] || @options[:twine_file]
132
+ write_twine_data(output_path)
133
+ end
134
+
135
+ def consume_all_localization_files
136
+ if !File.directory?(@options[:input_path])
137
+ raise Twine::Error.new("Directory does not exist: #{@options[:output_path]}")
138
+ end
139
+
140
+ Dir.glob(File.join(@options[:input_path], "**/*")) do |item|
141
+ if File.file?(item)
142
+ begin
143
+ read_localization_file(item)
144
+ rescue Twine::Error => e
145
+ Twine::stderr.puts "#{e.message}"
146
+ end
147
+ end
148
+ end
149
+
150
+ output_path = @options[:output_path] || @options[:twine_file]
151
+ write_twine_data(output_path)
152
+ end
153
+
154
+ def generate_loc_drop
155
+ validate_twine_file if @options[:validate]
156
+
157
+ require_rubyzip
158
+
159
+ if File.file?(@options[:output_path])
160
+ File.delete(@options[:output_path])
161
+ end
162
+
163
+ Dir.mktmpdir do |temp_dir|
164
+ Zip::File.open(@options[:output_path], Zip::File::CREATE) do |zipfile|
165
+ zipfile.mkdir('Locales')
166
+
167
+ formatter = formatter_for_format(@options[:format])
168
+ @twine_file.language_codes.each do |lang|
169
+ if @options[:languages] == nil || @options[:languages].length == 0 || @options[:languages].include?(lang)
170
+ file_name = lang + formatter.extension
171
+ temp_path = File.join(temp_dir, file_name)
172
+ zip_path = File.join('Locales', file_name)
173
+
174
+ output = formatter.format_file(lang)
175
+ unless output
176
+ Twine::stderr.puts "Skipping file #{file_name} since it would not contain any translations."
177
+ next
178
+ end
179
+
180
+ IO.write(temp_path, output, encoding: encoding)
181
+ zipfile.add(zip_path, temp_path)
182
+ end
183
+ end
184
+ end
185
+ end
186
+ end
187
+
188
+ def consume_loc_drop
189
+ require_rubyzip
190
+
191
+ if !File.file?(@options[:input_path])
192
+ raise Twine::Error.new("File does not exist: #{@options[:input_path]}")
193
+ end
194
+
195
+ Dir.mktmpdir do |temp_dir|
196
+ Zip::File.open(@options[:input_path]) do |zipfile|
197
+ zipfile.each do |entry|
198
+ next if entry.name.end_with? '/' or File.basename(entry.name).start_with? '.'
199
+
200
+ real_path = File.join(temp_dir, entry.name)
201
+ FileUtils.mkdir_p(File.dirname(real_path))
202
+ zipfile.extract(entry.name, real_path)
203
+ begin
204
+ read_localization_file(real_path)
205
+ rescue Twine::Error => e
206
+ Twine::stderr.puts "#{e.message}"
207
+ end
208
+ end
209
+ end
210
+ end
211
+
212
+ output_path = @options[:output_path] || @options[:twine_file]
213
+ write_twine_data(output_path)
214
+ end
215
+
216
+ def validate_twine_file
217
+ total_definitions = 0
218
+ all_keys = Set.new
219
+ duplicate_keys = Set.new
220
+ keys_without_tags = Set.new
221
+ invalid_keys = Set.new
222
+ valid_key_regex = /^[A-Za-z0-9_]+$/
223
+
224
+ @twine_file.sections.each do |section|
225
+ section.definitions.each do |definition|
226
+ total_definitions += 1
227
+
228
+ duplicate_keys.add(definition.key) if all_keys.include? definition.key
229
+ all_keys.add(definition.key)
230
+
231
+ keys_without_tags.add(definition.key) if definition.tags == nil or definition.tags.length == 0
232
+
233
+ invalid_keys << definition.key unless definition.key =~ valid_key_regex
234
+ end
235
+ end
236
+
237
+ errors = []
238
+ join_keys = lambda { |set| set.map { |k| " " + k }.join("\n") }
239
+
240
+ unless duplicate_keys.empty?
241
+ errors << "Found duplicate key(s):\n#{join_keys.call(duplicate_keys)}"
242
+ end
243
+
244
+ if @options[:pedantic]
245
+ if keys_without_tags.length == total_definitions
246
+ errors << "None of your definitions have tags."
247
+ elsif keys_without_tags.length > 0
248
+ errors << "Found definitions without tags:\n#{join_keys.call(keys_without_tags)}"
249
+ end
250
+ end
251
+
252
+ unless invalid_keys.empty?
253
+ errors << "Found key(s) with invalid characters:\n#{join_keys.call(invalid_keys)}"
254
+ end
255
+
256
+ raise Twine::Error.new errors.join("\n\n") unless errors.empty?
257
+
258
+ Twine::stdout.puts "#{@options[:twine_file]} is valid."
259
+ end
260
+
261
+ private
262
+
263
+ def encoding
264
+ @options[:output_encoding] || 'UTF-8'
265
+ end
266
+
267
+ def require_rubyzip
268
+ begin
269
+ require 'zip'
270
+ rescue LoadError
271
+ raise Twine::Error.new "You must run 'gem install rubyzip' in order to create or consume localization drops."
272
+ end
273
+ end
274
+
275
+ def determine_language_given_path(path)
276
+ code = File.basename(path, File.extname(path))
277
+ return code if @twine_file.language_codes.include? code
278
+ end
279
+
280
+ def formatter_for_format(format)
281
+ find_formatter { |f| f.format_name == format }
282
+ end
283
+
284
+ def find_formatter(&block)
285
+ formatter = Formatters.formatters.find &block
286
+ return nil unless formatter
287
+ formatter.twine_file = @twine_file
288
+ formatter.options = @options
289
+ formatter
290
+ end
291
+
292
+ def read_localization_file(path, lang = nil)
293
+ unless File.file?(path)
294
+ raise Twine::Error.new("File does not exist: #{path}")
295
+ end
296
+
297
+ formatter, lang = prepare_read_write(path, lang)
298
+
299
+ encoding = @options[:encoding] || Twine::Encoding.encoding_for_path(path)
300
+
301
+ IO.open(IO.sysopen(path, 'rb'), 'rb', external_encoding: encoding, internal_encoding: 'UTF-8') do |io|
302
+ io.read(2) if Twine::Encoding.has_bom?(path)
303
+ formatter.read(io, lang)
304
+ end
305
+ end
306
+
307
+ def prepare_read_write(path, lang)
308
+ formatter_for_path = find_formatter { |f| f.extension == File.extname(path) }
309
+ formatter = formatter_for_format(@options[:format]) || formatter_for_path
310
+
311
+ unless formatter
312
+ if !path.include?("Localizable.stringsdict")
313
+ raise Twine::Error.new "Unable to determine format of #{path}"
314
+ else
315
+ raise Twine::Error.new "Ignoring for now #{path}"
316
+ end
317
+ end
318
+
319
+ if formatter.can_handle_file?(path)
320
+ lang = lang || determine_language_given_path(path) || formatter.determine_language_given_path(path)
321
+ unless lang
322
+ raise Twine::Error.new "Unable to determine language for #{path}"
323
+ end
324
+
325
+ @twine_file.language_codes << lang unless @twine_file.language_codes.include? lang
326
+ else
327
+ raise Twine::Error.new "Ignoring #{path}"
328
+ end
329
+ return formatter, lang
330
+ end
331
+ end
332
+ end
@@ -0,0 +1,266 @@
1
+ module Twine
2
+ class TwineDefinition
3
+ attr_reader :key
4
+ attr_accessor :comment
5
+ attr_accessor :ios_comment
6
+ attr_accessor :tags
7
+ attr_reader :translations
8
+ attr_accessor :reference
9
+ attr_accessor :reference_key
10
+
11
+ def initialize(key)
12
+ @key = key
13
+ @comment = nil
14
+ @ios_comment = nil
15
+ @tags = nil
16
+ @translations = {}
17
+ end
18
+
19
+ def comment
20
+ raw_comment || (reference.comment if reference)
21
+ end
22
+
23
+ def raw_comment
24
+ @comment
25
+ end
26
+
27
+ def ios_comment
28
+ raw_ios_comment || (reference.ios_comment if reference)
29
+ end
30
+
31
+ def raw_ios_comment
32
+ @ios_comment
33
+ end
34
+
35
+ # [['tag1', 'tag2'], ['~tag3']] == (tag1 OR tag2) AND (!tag3)
36
+ def matches_tags?(tags, include_untagged)
37
+ if tags == nil || tags.empty? # The user did not specify any tags. Everything passes.
38
+ return true
39
+ elsif @tags == nil # This definition has no tags -> check reference (if any)
40
+ return reference ? reference.matches_tags?(tags, include_untagged) : include_untagged
41
+ elsif @tags.empty?
42
+ return include_untagged
43
+ else
44
+ return tags.all? do |set|
45
+ regular_tags, negated_tags = set.partition { |tag| tag[0] != '~' }
46
+ negated_tags.map! { |tag| tag[1..-1] }
47
+ matches_regular_tags = (!regular_tags.empty? && !(regular_tags & @tags).empty?)
48
+ matches_negated_tags = (!negated_tags.empty? && (negated_tags & @tags).empty?)
49
+ matches_regular_tags or matches_negated_tags
50
+ end
51
+ end
52
+
53
+ return false
54
+ end
55
+
56
+ def translation_for_lang(lang)
57
+ translation = [lang].flatten.map { |l| @translations[l] }.first
58
+
59
+ translation = reference.translation_for_lang(lang) if translation.nil? && reference
60
+
61
+ return translation
62
+ end
63
+
64
+ # Twine adds a copy of the dev language's translation if there is no definition provided for the language,
65
+ # which is useful in the main Localizable.strings file because iOS doesn't auto use the Base language's
66
+ # value, but we don't want that behaviour in our plurals
67
+ def translation_for_lang_or_nil(lang, dev_lang)
68
+ translation = [lang].flatten.map { |l| @translations[l] }.first
69
+
70
+ # translation never comes back as nil because Twine fills with the dev_lang string
71
+ if lang != dev_lang
72
+ [lang].flatten.map do |l|
73
+ if @translations[l] == @translations[dev_lang]
74
+ return nil
75
+ end
76
+ end
77
+ end
78
+
79
+ return translation
80
+ end
81
+ end
82
+
83
+ class TwineSection
84
+ attr_reader :name
85
+ attr_reader :definitions
86
+
87
+ def initialize(name)
88
+ @name = name
89
+ @definitions = []
90
+ end
91
+
92
+ def is_uncategorized
93
+ return @name == 'Uncategorized'
94
+ end
95
+ end
96
+
97
+ class TwineFile
98
+ attr_reader :sections
99
+ attr_reader :definitions_by_key
100
+ attr_reader :language_codes
101
+
102
+ private
103
+
104
+ def match_key(text)
105
+ match = /^\[(.+)\]$/.match(text)
106
+ return match[1] if match
107
+ end
108
+
109
+ public
110
+
111
+ def initialize
112
+ @sections = []
113
+ @definitions_by_key = {}
114
+ @language_codes = []
115
+ end
116
+
117
+ def add_language_code(code)
118
+ if @language_codes.length == 0
119
+ @language_codes << code
120
+ elsif !@language_codes.include?(code)
121
+ dev_lang = @language_codes[0]
122
+ @language_codes << code
123
+ @language_codes.delete(dev_lang)
124
+ @language_codes.sort!
125
+ @language_codes.insert(0, dev_lang)
126
+ end
127
+ end
128
+
129
+ def set_developer_language_code(code)
130
+ @language_codes.delete(code)
131
+ @language_codes.insert(0, code)
132
+ end
133
+
134
+ def read(path)
135
+ if !File.file?(path)
136
+ file = File.new(path, 'w:UTF-8')
137
+ file.close
138
+ end
139
+
140
+ File.open(path, 'r:UTF-8') do |f|
141
+ line_num = 0
142
+ current_section = nil
143
+ current_definition = nil
144
+ while line = f.gets
145
+ parsed = false
146
+ line.strip!
147
+ line_num += 1
148
+
149
+ if line.length == 0
150
+ next
151
+ end
152
+
153
+ if line.length > 4 && line[0, 2] == '[['
154
+ match = /^\[\[(.+)\]\]$/.match(line)
155
+ if match
156
+ current_section = TwineSection.new(match[1])
157
+ @sections << current_section
158
+ parsed = true
159
+ end
160
+ elsif line.length > 2 && line[0, 1] == '['
161
+ key = match_key(line)
162
+ if key
163
+ current_definition = TwineDefinition.new(key)
164
+ @definitions_by_key[current_definition.key] = current_definition
165
+ if !current_section
166
+ current_section = TwineSection.new('')
167
+ @sections << current_section
168
+ end
169
+ current_section.definitions << current_definition
170
+ parsed = true
171
+ end
172
+ else
173
+ match = /^([^=]+)=(.*)$/.match(line)
174
+ if match
175
+ key = match[1].strip
176
+ value = match[2].strip
177
+
178
+ value = value[1..-2] if value[0] == '`' && value[-1] == '`'
179
+ case key
180
+ when 'ios'
181
+ current_definition.ios_comment = value
182
+ when 'comment'
183
+ current_definition.comment = value
184
+ when 'tags'
185
+ current_definition.tags = value.split(',')
186
+ when 'ref'
187
+ current_definition.reference_key = value if value
188
+ else
189
+ if !@language_codes.include? key
190
+ add_language_code(key)
191
+ end
192
+ current_definition.translations[key] = value
193
+ end
194
+ parsed = true
195
+ end
196
+ end
197
+
198
+ if !parsed
199
+ raise Twine::Error.new("Unable to parse line #{line_num} of #{path}: #{line}")
200
+ end
201
+ end
202
+ end
203
+
204
+ # resolve_references
205
+ @definitions_by_key.each do |key, definition|
206
+ next unless definition.reference_key
207
+ definition.reference = @definitions_by_key[definition.reference_key]
208
+ end
209
+ end
210
+
211
+ def write(path)
212
+ dev_lang = @language_codes[0]
213
+
214
+ File.open(path, 'w:UTF-8') do |f|
215
+ @sections.each do |section|
216
+ if f.pos > 0
217
+ f.puts ''
218
+ end
219
+
220
+ f.puts "[[#{section.name}]]"
221
+
222
+ section.definitions.each do |definition|
223
+ f.puts "\t[#{definition.key}]"
224
+
225
+ value = write_value(definition, dev_lang, f)
226
+ if !value && !definition.reference_key
227
+ puts "Warning: #{definition.key} does not exist in developer language '#{dev_lang}'"
228
+ end
229
+
230
+ if definition.reference_key
231
+ f.puts "\t\tref = #{definition.reference_key}"
232
+ end
233
+ if definition.tags && definition.tags.length > 0
234
+ tag_str = definition.tags.join(',')
235
+ f.puts "\t\ttags = #{tag_str}"
236
+ end
237
+ if definition.raw_comment and definition.raw_comment.length > 0
238
+ f.puts "\t\tcomment = #{definition.raw_comment}"
239
+ end
240
+ if definition.raw_ios_comment and definition.raw_ios_comment.length > 0
241
+ f.puts "\t\tios = #{definition.raw_ios_comment}"
242
+ end
243
+ @language_codes[1..-1].each do |lang|
244
+ write_value(definition, lang, f)
245
+ end
246
+ end
247
+ end
248
+ end
249
+ end
250
+
251
+ private
252
+
253
+ def write_value(definition, language, file)
254
+ value = definition.translations[language]
255
+ return nil unless value
256
+
257
+ if value[0] == ' ' || value[-1] == ' ' || (value[0] == '`' && value[-1] == '`')
258
+ value = '`' + value + '`'
259
+ end
260
+
261
+ file.puts "\t\t#{language} = #{value}"
262
+ return value
263
+ end
264
+
265
+ end
266
+ end