yury-twine 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +2 -0
  3. data/LICENSE +30 -0
  4. data/README.md +230 -0
  5. data/bin/twine +7 -0
  6. data/lib/twine.rb +36 -0
  7. data/lib/twine/cli.rb +200 -0
  8. data/lib/twine/encoding.rb +22 -0
  9. data/lib/twine/formatters.rb +20 -0
  10. data/lib/twine/formatters/abstract.rb +187 -0
  11. data/lib/twine/formatters/android.rb +254 -0
  12. data/lib/twine/formatters/apple.rb +328 -0
  13. data/lib/twine/output_processor.rb +57 -0
  14. data/lib/twine/placeholders.rb +54 -0
  15. data/lib/twine/plugin.rb +62 -0
  16. data/lib/twine/runner.rb +332 -0
  17. data/lib/twine/twine_file.rb +266 -0
  18. data/lib/twine/version.rb +3 -0
  19. data/test/command_test.rb +14 -0
  20. data/test/fixtures/consume_loc_drop.zip +0 -0
  21. data/test/fixtures/enc_utf16be.dummy +0 -0
  22. data/test/fixtures/enc_utf16be_bom.dummy +0 -0
  23. data/test/fixtures/enc_utf16le.dummy +0 -0
  24. data/test/fixtures/enc_utf16le_bom.dummy +0 -0
  25. data/test/fixtures/enc_utf8.dummy +2 -0
  26. data/test/fixtures/formatter_android.xml +15 -0
  27. data/test/fixtures/formatter_apple.strings +20 -0
  28. data/test/fixtures/formatter_django.po +30 -0
  29. data/test/fixtures/formatter_flash.properties +15 -0
  30. data/test/fixtures/formatter_gettext.po +26 -0
  31. data/test/fixtures/formatter_jquery.json +7 -0
  32. data/test/fixtures/formatter_tizen.xml +15 -0
  33. data/test/fixtures/gettext_multiline.po +10 -0
  34. data/test/fixtures/twine_accent_values.txt +13 -0
  35. data/test/test_abstract_formatter.rb +165 -0
  36. data/test/test_cli.rb +304 -0
  37. data/test/test_consume_loc_drop.rb +27 -0
  38. data/test/test_consume_localization_file.rb +119 -0
  39. data/test/test_formatters.rb +363 -0
  40. data/test/test_generate_all_localization_files.rb +102 -0
  41. data/test/test_generate_loc_drop.rb +80 -0
  42. data/test/test_generate_localization_file.rb +91 -0
  43. data/test/test_output_processor.rb +85 -0
  44. data/test/test_placeholders.rb +84 -0
  45. data/test/test_twine_definition.rb +111 -0
  46. data/test/test_twine_file.rb +58 -0
  47. data/test/test_validate_twine_file.rb +61 -0
  48. data/test/twine_file_dsl.rb +46 -0
  49. data/test/twine_test.rb +48 -0
  50. metadata +179 -0
@@ -0,0 +1,328 @@
1
+ require 'Nokogiri'
2
+
3
+ module Twine
4
+ module Formatters
5
+ class Apple < Abstract
6
+ def format_name
7
+ 'apple'
8
+ end
9
+
10
+ def extension
11
+ '.strings'
12
+ end
13
+
14
+ def can_handle_directory?(path)
15
+ Dir.entries(path).any? { |item| /^.+\.lproj$/.match(item) }
16
+ end
17
+
18
+ def can_handle_file?(path)
19
+ path_arr = path.split(File::SEPARATOR)
20
+ file_name = path_arr[path_arr.length - 1]
21
+ return file_name == default_file_name || file_name == default_plural_file_name
22
+ end
23
+
24
+ def default_file_name
25
+ return 'Localizable.strings'
26
+ end
27
+
28
+ def default_plural_file_name
29
+ return 'Localizable.stringsdict'
30
+ end
31
+
32
+ def determine_language_given_path(path)
33
+ path_arr = path.split(File::SEPARATOR)
34
+ path_arr.each do |segment|
35
+ match = /^(.+)\.lproj$/.match(segment)
36
+ if match
37
+ if match[1] != "Base"
38
+ return match[1]
39
+ else
40
+ return 'en'
41
+ end
42
+ end
43
+ end
44
+
45
+ return
46
+ end
47
+
48
+ def output_path_for_language(lang)
49
+ if lang == 'en'
50
+ "Base.lproj"
51
+ else
52
+ "#{lang}.lproj"
53
+ end
54
+ end
55
+
56
+ def read(io, lang)
57
+ uncategorized_section = nil
58
+ if !section_exists('Uncategorized')
59
+ uncategorized_section = TwineSection.new('Uncategorized')
60
+ @twine_file.sections.insert(0, uncategorized_section)
61
+ else
62
+ uncategorized_section = get_section('Uncategorized')
63
+ end
64
+ last_comment = nil
65
+ while line = io.gets
66
+ # matches a `key = "value"` line, where key may be quoted or unquoted. The former may also contain escaped characters
67
+ match = /^\s*((?:"(?:[^"\\]|\\.)+")|(?:[^"\s=]+))\s*=\s*"((?:[^"\\]|\\.)*)"/.match(line)
68
+ if match
69
+ key = match[1]
70
+ key = key[1..-2] if key[0] == '"' and key[-1] == '"'
71
+ key.gsub!('\\"', '"')
72
+ value = match[2]
73
+ value.gsub!('\\"', '"')
74
+ value.gsub!('%s', '%@')
75
+ value.gsub!('$s', '$@')
76
+ set_translation_for_key(uncategorized_section, key, lang, value)
77
+ if last_comment
78
+ set_comment_for_key(key, last_comment)
79
+ end
80
+ end
81
+
82
+ match = /\/\* (.*) \*\//.match(line)
83
+ if match
84
+ last_comment = match[1]
85
+ else
86
+ last_comment = nil
87
+ end
88
+ end
89
+
90
+ # Handle plural files
91
+ if (!File.file?(plural_input_file_for_lang(lang)))
92
+ return
93
+ end
94
+
95
+ doc = File.open(plural_input_file_for_lang(lang)) { |f| Nokogiri::XML(f) }
96
+ comment = nil
97
+ key = nil
98
+ value = nil
99
+
100
+ top_level_dict = doc.css("dict").first
101
+ whole_dicts = top_level_dict.xpath("./dict")
102
+ plural_keys = top_level_dict.xpath("./key")
103
+
104
+ for i in 0 ... whole_dicts.size do
105
+ current_dict = whole_dicts[i]
106
+ comment = current_dict.css("string").first.content
107
+ key = plural_keys[i].content.to_s
108
+
109
+ nested_dict = current_dict.xpath("./dict")
110
+ nested_dict_keys = nested_dict.xpath("./key")
111
+ nested_dict_strings = nested_dict.xpath("./string")
112
+
113
+ section = nil
114
+ if !section_exists(key)
115
+ section = TwineSection.new(key)
116
+ @twine_file.sections.insert(@twine_file.sections.size - 1, section)
117
+ else
118
+ section = get_section(key)
119
+ end
120
+
121
+ for j in 0 ... nested_dict_keys.children.size do
122
+ cur_xml_key = nested_dict_keys.children[j]
123
+ if !cur_xml_key.content.include? "NSString"
124
+ modified_key = key + "__" + cur_xml_key.content.to_s
125
+
126
+ set_translation_for_key(section, modified_key, lang, nested_dict_strings[j].content.to_s)
127
+ set_ios_comment_for_key(modified_key, comment)
128
+ end
129
+ end
130
+ end
131
+ end
132
+
133
+ def plural_input_file_for_lang(lang)
134
+ path = @options[:input_path]
135
+ if path.include?(output_path_for_language(lang))
136
+ if path.include?(default_file_name)
137
+ path.sub(default_file_name, default_plural_file_name)
138
+ else
139
+ path + "/" + default_plural_file_name
140
+ end
141
+ else
142
+ path + output_path_for_language(lang) + "/" + default_plural_file_name
143
+ end
144
+ end
145
+
146
+ def plural_output_file_for_lang(lang)
147
+ path = @options[:output_path]
148
+ if path.include?(output_path_for_language(lang))
149
+ if path.include?(default_file_name)
150
+ path.sub(default_file_name, default_plural_file_name)
151
+ else
152
+ path + "/" + default_plural_file_name
153
+ end
154
+ else
155
+ path + output_path_for_language(lang) + "/" + default_plural_file_name
156
+ end
157
+ end
158
+
159
+ def format_sections(twine_file, lang)
160
+ first_plural = true
161
+ out_file = File.open(plural_output_file_for_lang(lang), "w")
162
+
163
+ sections = Array.new(twine_file.sections.size)
164
+ for i in 0 ... twine_file.sections.size
165
+ section = twine_file.sections[i]
166
+ if section.is_uncategorized
167
+ sections[i] = format_section(section, lang)
168
+ else
169
+ if first_plural
170
+ first_plural = false
171
+ out_file.puts(format_header_stringsdict)
172
+ end
173
+ format_section_plural(section, lang, out_file)
174
+ end
175
+ end
176
+ out_file.puts(format_footer_stringsdict)
177
+ out_file.close
178
+ sections.compact.join("\n")
179
+ end
180
+
181
+ def format_section(section, lang)
182
+ definitions = section.definitions.select { |definition| should_include_definition(definition, lang) }
183
+ return if definitions.empty?
184
+
185
+ result = ""
186
+
187
+ if section.name && section.name.length > 0
188
+ section_header = format_section_header(section)
189
+ result += "\n#{section_header}" if section_header
190
+
191
+ if section.is_uncategorized
192
+ definitions.map! { |definition| format_definition(definition, lang) }
193
+ definitions.compact! # remove nil definitions
194
+ definitions.map! { |definition| "\n#{definition}" } # prepend newline
195
+ result += definitions.join
196
+ end
197
+ end
198
+ end
199
+
200
+ def format_section_plural(section, lang, out_file)
201
+ plural_key = section.name
202
+ main_file_contains_key = main_localizable_file_contains_key(plural_key)
203
+
204
+ for i in 0 ... section.definitions.size
205
+ definition = section.definitions[i]
206
+ value = definition.translation_for_lang_or_nil(lang, @twine_file.language_codes[0])
207
+ ios_localized_format_key = definition.ios_comment
208
+
209
+ if ios_localized_format_key == nil
210
+ puts "[" + plural_key + "]"
211
+
212
+ if main_file_contains_key
213
+ puts "Needs matching key in Localizable.strings"
214
+ else
215
+ puts "This is an Android-only plural"
216
+ end
217
+ return
218
+ end
219
+
220
+ if i == 0 && main_file_contains_key
221
+ out_file.puts(format_plural_start(plural_key, ios_localized_format_key.to_s))
222
+ end
223
+
224
+ if value != nil && main_file_contains_key
225
+ out_file.puts(format_plural_key_value(definition.key, format_value_plural(value.dup)))
226
+ end
227
+ end
228
+
229
+ if main_file_contains_key
230
+ out_file.puts(format_plural_section_end)
231
+ end
232
+ end
233
+
234
+ def main_localizable_file_contains_key(key)
235
+ @twine_file.sections.each do |section|
236
+ if section.is_uncategorized
237
+ section.definitions.each do |definition|
238
+ if definition.key == key
239
+ return true
240
+ end
241
+ end
242
+ return false
243
+ end
244
+ end
245
+ # Should never reach here
246
+ return false
247
+ end
248
+
249
+ ########### PLURALS START ###########
250
+
251
+ def format_header_stringsdict
252
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
253
+ "<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n" +
254
+ "<plist version=\"1.0\">\n" +
255
+ " <dict>\n"
256
+ end
257
+
258
+ def format_plural_start(section_name, comment)
259
+ comment_parts = comment.split("@")
260
+
261
+ # TODO: Should this be hardcoded or using Nokogiri somehow?
262
+ " <key>#{section_name}</key>\n" +
263
+ " <dict>\n" +
264
+ " <key>NSStringLocalizedFormatKey</key>\n" +
265
+ " <string>#{comment}</string>\n" +
266
+ " <key>#{comment_parts[1]}</key>\n" +
267
+ " <dict>\n" +
268
+ " <key>NSStringFormatSpecTypeKey</key>\n" +
269
+ " <string>NSStringPluralRuleType</string>\n" +
270
+ " <key>NSStringFormatValueTypeKey</key>\n" +
271
+ " <string>d</string>\n"
272
+ end
273
+
274
+ def format_plural_key_value(plural_key, plural_string)
275
+ plural_key_parts = plural_key.rpartition(/.__/)
276
+ # TODO: Nokogiri?
277
+ " <key>#{plural_key_parts[plural_key_parts.length - 1]}</key>\n" +
278
+ " <string>#{plural_string}</string>\n"
279
+ end
280
+
281
+ def format_plural_section_end
282
+ # TODO: Nokogiri?
283
+ " </dict>\n" +
284
+ " </dict>\n"
285
+ end
286
+
287
+ def format_footer_stringsdict
288
+ # TODO: Nokogiri?
289
+ " </dict>\n" +
290
+ "</plist>"
291
+ end
292
+
293
+ ########### PLURALS END ###########
294
+
295
+ def format_header(lang)
296
+ "/**\n * Apple Strings File\n * Generated by Twine #{Twine::VERSION}\n * Language: #{lang}\n */"
297
+ end
298
+
299
+ def format_section_header(section)
300
+ "/********** #{section.name} **********/\n"
301
+ end
302
+
303
+ def key_value_pattern
304
+ "\"%{key}\" = \"%{value}\";\n"
305
+ end
306
+
307
+ def format_comment(definition, lang)
308
+ "/* #{definition.comment.gsub('*/', '* /')} */\n" if definition.comment
309
+ end
310
+
311
+ def format_key(key)
312
+ escape_quotes(key)
313
+ end
314
+
315
+ def format_value(value)
316
+ text = escape_quotes(value)
317
+ text.gsub("b>", "strong>")
318
+ end
319
+
320
+ def format_value_plural(value)
321
+ text = escape_quotes(value)
322
+ text.gsub("b>", "strong&gt;").gsub("<", "&lt;")
323
+ end
324
+ end
325
+ end
326
+ end
327
+
328
+ Twine::Formatters.formatters << Twine::Formatters::Apple.new
@@ -0,0 +1,57 @@
1
+ module Twine
2
+ module Processors
3
+
4
+ class OutputProcessor
5
+ def initialize(twine_file, options)
6
+ @twine_file = twine_file
7
+ @options = options
8
+ end
9
+
10
+ def default_language
11
+ @options[:developer_language] || @twine_file.language_codes[0]
12
+ end
13
+
14
+ def fallback_languages(language)
15
+ fallback_mapping = {
16
+ 'zh-TW' => 'zh-Hant' # if we don't have a zh-TW translation, try zh-Hant before en
17
+ }
18
+
19
+ [fallback_mapping[language], default_language].flatten.compact
20
+ end
21
+
22
+ def process(language)
23
+ result = TwineFile.new
24
+
25
+ result.language_codes.concat @twine_file.language_codes
26
+ @twine_file.sections.each do |section|
27
+ new_section = TwineSection.new section.name
28
+
29
+ section.definitions.each do |definition|
30
+ next unless definition.matches_tags?(@options[:tags], @options[:untagged])
31
+
32
+ value = definition.translation_for_lang(language)
33
+
34
+ next if value && @options[:include] == :untranslated
35
+
36
+ if value.nil? && @options[:include] != :translated
37
+ value = definition.translation_for_lang(fallback_languages(language))
38
+ end
39
+
40
+ next unless value
41
+
42
+ new_definition = definition.dup
43
+ new_definition.translations[language] = value
44
+
45
+ new_section.definitions << new_definition
46
+ result.definitions_by_key[new_definition.key] = new_definition
47
+ end
48
+
49
+ result.sections << new_section
50
+ end
51
+
52
+ return result
53
+ end
54
+ end
55
+
56
+ end
57
+ end
@@ -0,0 +1,54 @@
1
+ module Twine
2
+ module Placeholders
3
+ extend self
4
+
5
+ PLACEHOLDER_FLAGS_WIDTH_PRECISION_LENGTH = '([-+ 0#])?(\d+|\*)?(\.(\d+|\*))?(hh?|ll?|L|z|j|t)?'
6
+ PLACEHOLDER_PARAMETER_FLAGS_WIDTH_PRECISION_LENGTH = '(\d+\$)?' + PLACEHOLDER_FLAGS_WIDTH_PRECISION_LENGTH
7
+
8
+ # http://developer.android.com/guide/topics/resources/string-resource.html#FormattingAndStyling
9
+ # http://stackoverflow.com/questions/4414389/android-xml-percent-symbol
10
+ # https://github.com/mobiata/twine/pull/106
11
+ def convert_placeholders_from_twine_to_android(input)
12
+ placeholder_types = '[diufFeEgGxXoscpaA]'
13
+
14
+ # %@ -> %s
15
+ value = input.gsub(/(%#{PLACEHOLDER_PARAMETER_FLAGS_WIDTH_PRECISION_LENGTH})@/, '\1s')
16
+
17
+ placeholder_syntax = PLACEHOLDER_PARAMETER_FLAGS_WIDTH_PRECISION_LENGTH + placeholder_types
18
+ placeholder_regex = /%#{placeholder_syntax}/
19
+
20
+ number_of_placeholders = value.scan(placeholder_regex).size
21
+
22
+ return value if number_of_placeholders == 0
23
+
24
+ # got placeholders -> need to double single percent signs
25
+ # % -> %% (but %% -> %%, %d -> %d)
26
+ single_percent_regex = /([^%])(%)(?!(%|#{placeholder_syntax}))/
27
+ value.gsub! single_percent_regex, '\1%%'
28
+
29
+ return value if number_of_placeholders < 2
30
+
31
+ # number placeholders
32
+ non_numbered_placeholder_regex = /%(#{PLACEHOLDER_FLAGS_WIDTH_PRECISION_LENGTH}#{placeholder_types})/
33
+
34
+ number_of_non_numbered_placeholders = value.scan(non_numbered_placeholder_regex).size
35
+
36
+ return value if number_of_non_numbered_placeholders == 0
37
+
38
+ raise Twine::Error.new("The value \"#{input}\" contains numbered and non-numbered placeholders") if number_of_placeholders != number_of_non_numbered_placeholders
39
+
40
+ # %d -> %$1d
41
+ index = 0
42
+ value.gsub!(non_numbered_placeholder_regex) { "%#{index += 1}$#{$1}" }
43
+
44
+ value
45
+ end
46
+
47
+ def convert_placeholders_from_android_to_twine(input)
48
+ placeholder_regex = /(%#{PLACEHOLDER_PARAMETER_FLAGS_WIDTH_PRECISION_LENGTH})s/
49
+
50
+ # %s -> %@
51
+ input.gsub(placeholder_regex, '\1@')
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,62 @@
1
+ require 'safe_yaml/load'
2
+
3
+ SafeYAML::OPTIONS[:suppress_warnings] = true
4
+
5
+ module Twine
6
+ class Plugin
7
+ attr_reader :debug, :config
8
+
9
+ def initialize
10
+ @debug = false
11
+ require_gems
12
+ end
13
+
14
+ ###
15
+ # require gems from the yaml config.
16
+ #
17
+ # gems: [twine-plugin1, twine-2]
18
+ #
19
+ # also works with single gem
20
+ #
21
+ # gems: twine-plugin1
22
+ #
23
+ def require_gems
24
+ # ./twine.yml # current working directory
25
+ # ~/.twine # home directory
26
+ # /etc/twine.yml # etc
27
+ cwd_config = join_path Dir.pwd, 'twine.yml'
28
+ home_config = join_path Dir.home, '.twine'
29
+ etc_config = '/etc/twine.yml'
30
+
31
+ config_order = [cwd_config, home_config, etc_config]
32
+
33
+ puts "Config order: #{config_order}" if debug
34
+
35
+ config_order.each do |config_file|
36
+ next unless valid_file config_file
37
+ puts "Loading: #{config_file}" if debug
38
+ @config = SafeYAML.load_file config_file
39
+ puts "Config yaml: #{config}" if debug
40
+ break
41
+ end
42
+
43
+ return unless config
44
+
45
+ # wrap gems in an array. if nil then array will be empty
46
+ Kernel.Array(config['gems']).each do |gem_path|
47
+ puts "Requiring: #{gem_path}" if debug
48
+ require gem_path
49
+ end
50
+ end
51
+
52
+ private
53
+
54
+ def valid_file path
55
+ File.exist?(path) && File.readable?(path) && !File.directory?(path)
56
+ end
57
+
58
+ def join_path *paths
59
+ File.expand_path File.join *paths
60
+ end
61
+ end
62
+ end