yury-twine 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +2 -0
  3. data/LICENSE +30 -0
  4. data/README.md +230 -0
  5. data/bin/twine +7 -0
  6. data/lib/twine.rb +36 -0
  7. data/lib/twine/cli.rb +200 -0
  8. data/lib/twine/encoding.rb +22 -0
  9. data/lib/twine/formatters.rb +20 -0
  10. data/lib/twine/formatters/abstract.rb +187 -0
  11. data/lib/twine/formatters/android.rb +254 -0
  12. data/lib/twine/formatters/apple.rb +328 -0
  13. data/lib/twine/output_processor.rb +57 -0
  14. data/lib/twine/placeholders.rb +54 -0
  15. data/lib/twine/plugin.rb +62 -0
  16. data/lib/twine/runner.rb +332 -0
  17. data/lib/twine/twine_file.rb +266 -0
  18. data/lib/twine/version.rb +3 -0
  19. data/test/command_test.rb +14 -0
  20. data/test/fixtures/consume_loc_drop.zip +0 -0
  21. data/test/fixtures/enc_utf16be.dummy +0 -0
  22. data/test/fixtures/enc_utf16be_bom.dummy +0 -0
  23. data/test/fixtures/enc_utf16le.dummy +0 -0
  24. data/test/fixtures/enc_utf16le_bom.dummy +0 -0
  25. data/test/fixtures/enc_utf8.dummy +2 -0
  26. data/test/fixtures/formatter_android.xml +15 -0
  27. data/test/fixtures/formatter_apple.strings +20 -0
  28. data/test/fixtures/formatter_django.po +30 -0
  29. data/test/fixtures/formatter_flash.properties +15 -0
  30. data/test/fixtures/formatter_gettext.po +26 -0
  31. data/test/fixtures/formatter_jquery.json +7 -0
  32. data/test/fixtures/formatter_tizen.xml +15 -0
  33. data/test/fixtures/gettext_multiline.po +10 -0
  34. data/test/fixtures/twine_accent_values.txt +13 -0
  35. data/test/test_abstract_formatter.rb +165 -0
  36. data/test/test_cli.rb +304 -0
  37. data/test/test_consume_loc_drop.rb +27 -0
  38. data/test/test_consume_localization_file.rb +119 -0
  39. data/test/test_formatters.rb +363 -0
  40. data/test/test_generate_all_localization_files.rb +102 -0
  41. data/test/test_generate_loc_drop.rb +80 -0
  42. data/test/test_generate_localization_file.rb +91 -0
  43. data/test/test_output_processor.rb +85 -0
  44. data/test/test_placeholders.rb +84 -0
  45. data/test/test_twine_definition.rb +111 -0
  46. data/test/test_twine_file.rb +58 -0
  47. data/test/test_validate_twine_file.rb +61 -0
  48. data/test/twine_file_dsl.rb +46 -0
  49. data/test/twine_test.rb +48 -0
  50. metadata +179 -0
@@ -0,0 +1,22 @@
1
+ module Twine
2
+ module Encoding
3
+
4
+ def self.bom(path)
5
+ first_bytes = IO.binread(path, 2)
6
+ return nil unless first_bytes
7
+ first_bytes = first_bytes.codepoints.map.to_a
8
+ return 'UTF-16BE' if first_bytes == [0xFE, 0xFF]
9
+ return 'UTF-16LE' if first_bytes == [0xFF, 0xFE]
10
+ rescue EOFError
11
+ return nil
12
+ end
13
+
14
+ def self.has_bom?(path)
15
+ !bom(path).nil?
16
+ end
17
+
18
+ def self.encoding_for_path(path)
19
+ bom(path) || 'UTF-8'
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,20 @@
1
+ module Twine
2
+ module Formatters
3
+ @formatters = []
4
+
5
+ class << self
6
+ attr_reader :formatters
7
+
8
+ ###
9
+ # registers a new formatter
10
+ #
11
+ # formatter_class - the class of the formatter to register
12
+ #
13
+ # returns array of active formatters
14
+ #
15
+ def register_formatter formatter_class
16
+ @formatters << formatter_class.new
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,187 @@
1
+ require 'fileutils'
2
+
3
+ module Twine
4
+ module Formatters
5
+ class Abstract
6
+ attr_accessor :twine_file
7
+ attr_accessor :options
8
+
9
+ def initialize
10
+ @twine_file = TwineFile.new
11
+ @options = {}
12
+ end
13
+
14
+ def format_name
15
+ raise NotImplementedError.new("You must implement format_name in your formatter class.")
16
+ end
17
+
18
+ def extension
19
+ raise NotImplementedError.new("You must implement extension in your formatter class.")
20
+ end
21
+
22
+ def can_handle_directory?(path)
23
+ raise NotImplementedError.new("You must implement can_handle_directory? in your formatter class.")
24
+ end
25
+
26
+ def can_handle_file?(path)
27
+ raise NotImplementedError.new("You must implement can_handle_file? in your formatter class.")
28
+ end
29
+
30
+ def default_file_name
31
+ raise NotImplementedError.new("You must implement default_file_name in your formatter class.")
32
+ end
33
+
34
+ def set_translation_for_key(current_section, key, lang, value)
35
+ value = value.gsub("\n", "\\n")
36
+
37
+ if @twine_file.definitions_by_key.include?(key)
38
+ definition = @twine_file.definitions_by_key[key]
39
+ reference = @twine_file.definitions_by_key[definition.reference_key] if definition.reference_key
40
+
41
+ if !reference or value != reference.translations[lang]
42
+ definition.translations[lang] = value
43
+ end
44
+ elsif @options[:consume_all]
45
+ Twine::stderr.puts "Adding new definition '#{key}' to twine file."
46
+ current_definition = TwineDefinition.new(key)
47
+ current_section.definitions << current_definition
48
+
49
+ if @options[:tags] && @options[:tags].length > 0
50
+ current_definition.tags = @options[:tags]
51
+ end
52
+
53
+ @twine_file.definitions_by_key[key] = current_definition
54
+ @twine_file.definitions_by_key[key].translations[lang] = value
55
+ else
56
+ Twine::stderr.puts "Warning: '#{key}' not found in twine file."
57
+ end
58
+ if !@twine_file.language_codes.include?(lang)
59
+ @twine_file.add_language_code(lang)
60
+ end
61
+ end
62
+
63
+ def set_comment_for_key(key, comment)
64
+ return unless @options[:consume_comments]
65
+
66
+ if @twine_file.definitions_by_key.include?(key)
67
+ definition = @twine_file.definitions_by_key[key]
68
+
69
+ reference = @twine_file.definitions_by_key[definition.reference_key] if definition.reference_key
70
+
71
+ if !reference or comment != reference.raw_comment
72
+ definition.comment = comment
73
+ end
74
+ end
75
+ end
76
+
77
+ def set_ios_comment_for_key(key, comment)
78
+ if @twine_file.definitions_by_key.include?(key)
79
+ definition = @twine_file.definitions_by_key[key]
80
+
81
+ reference = @twine_file.definitions_by_key[definition.reference_key] if definition.reference_key
82
+
83
+ if !reference or ios_comment != reference.raw_ios_comment
84
+ definition.ios_comment = comment
85
+ end
86
+ end
87
+ end
88
+
89
+ def determine_language_given_path(path)
90
+ raise NotImplementedError.new("You must implement determine_language_given_path in your formatter class.")
91
+ end
92
+
93
+ def output_path_for_language(lang)
94
+ lang
95
+ end
96
+
97
+ def read(io, lang)
98
+ raise NotImplementedError.new("You must implement read in your formatter class.")
99
+ end
100
+
101
+ def format_file(lang)
102
+ output_processor = Processors::OutputProcessor.new(@twine_file, @options)
103
+ processed_twine_file = output_processor.process(lang)
104
+
105
+ return nil if processed_twine_file.definitions_by_key.empty?
106
+
107
+ header = format_header(lang)
108
+ result = ""
109
+ result += header + "\n" if header
110
+ result += format_sections(processed_twine_file, lang)
111
+ end
112
+
113
+ def format_header(lang)
114
+ end
115
+
116
+ def format_sections(twine_file, lang)
117
+ sections = twine_file.sections.map { |section| format_section(section, lang) }
118
+ sections.compact.join("\n")
119
+ end
120
+
121
+ def format_section_header(section)
122
+ end
123
+
124
+ def should_include_definition(definition, lang)
125
+ return !definition.translation_for_lang(lang).nil?
126
+ end
127
+
128
+ def format_section(section, lang)
129
+ definitions = section.definitions.select { |definition| should_include_definition(definition, lang) }
130
+ return if definitions.empty?
131
+
132
+ result = ""
133
+
134
+ if section.name && section.name.length > 0
135
+ section_header = format_section_header(section)
136
+ result += "\n#{section_header}" if section_header
137
+ end
138
+
139
+ definitions.map! { |definition| format_definition(definition, lang) }
140
+ definitions.compact! # remove nil definitions
141
+ definitions.map! { |definition| "\n#{definition}" } # prepend newline
142
+ result += definitions.join
143
+ end
144
+
145
+ def format_definition(definition, lang)
146
+ [format_comment(definition, lang), format_key_value(definition, lang)].compact.join
147
+ end
148
+
149
+ def format_comment(definition, lang)
150
+ end
151
+
152
+ def format_key_value(definition, lang)
153
+ value = definition.translation_for_lang(lang)
154
+ key_value_pattern % { key: format_key(definition.key.dup), value: format_value(value.dup) }
155
+ end
156
+
157
+ def key_value_pattern
158
+ raise NotImplementedError.new("You must implement key_value_pattern in your formatter class.")
159
+ end
160
+
161
+ def format_key(key)
162
+ key
163
+ end
164
+
165
+ def format_value(value)
166
+ value
167
+ end
168
+
169
+ def escape_quotes(text)
170
+ text.gsub('"', "\\\"")
171
+ end
172
+
173
+ def section_exists(section_name)
174
+ @twine_file.sections.find { |s| s.name == section_name }
175
+ end
176
+
177
+ def get_section(name)
178
+ @twine_file.sections.each do |s|
179
+ if s.name == name
180
+ return s
181
+ end
182
+ end
183
+ return nil
184
+ end
185
+ end
186
+ end
187
+ end
@@ -0,0 +1,254 @@
1
+ # encoding: utf-8
2
+ require 'cgi'
3
+ require 'rexml/document'
4
+
5
+ module Twine
6
+ module Formatters
7
+ class Android < Abstract
8
+ include Twine::Placeholders
9
+
10
+ LANG_MAPPINGS = Hash[
11
+ 'zh-rCN' => 'zh-Hans',
12
+ 'zh-rHK' => 'zh-Hant',
13
+ 'en-rGB' => 'en-UK',
14
+ 'in' => 'id',
15
+ 'nb' => 'no'
16
+ # TODO: spanish
17
+ ]
18
+
19
+ def format_name
20
+ 'android'
21
+ end
22
+
23
+ def extension
24
+ '.xml'
25
+ end
26
+
27
+ def can_handle_directory?(path)
28
+ Dir.entries(path).any? { |item| /^values.*$/.match(item) }
29
+ end
30
+
31
+ def can_handle_file?(path)
32
+ path_arr = path.split(File::SEPARATOR)
33
+ return path_arr[path_arr.length - 1] == default_file_name
34
+ end
35
+
36
+ def default_file_name
37
+ return 'strings.xml'
38
+ end
39
+
40
+ def determine_language_given_path(path)
41
+ path_arr = path.split(File::SEPARATOR)
42
+ path_arr.each do |segment|
43
+ if segment == 'values'
44
+ return 'en'
45
+ else
46
+ # The language is defined by a two-letter ISO 639-1 language code, optionally followed by a two letter ISO 3166-1-alpha-2 region code (preceded by lowercase "r").
47
+ # see http://developer.android.com/guide/topics/resources/providing-resources.html#AlternativeResources
48
+ match = /^values-([a-z]{2}(-r[a-z]{2})?)$/i.match(segment)
49
+ if match
50
+ lang = match[1]
51
+ lang = LANG_MAPPINGS.fetch(lang, lang)
52
+ lang.sub!('-r', '-')
53
+ return lang
54
+ end
55
+ end
56
+ end
57
+
58
+ return
59
+ end
60
+
61
+ def output_path_for_language(lang)
62
+ if lang == 'en'
63
+ "values"
64
+ else
65
+ "values-" + (LANG_MAPPINGS.key(lang) || lang)
66
+ end
67
+ end
68
+
69
+ def set_translation_for_key(section, key, lang, value)
70
+ value = CGI.unescapeHTML(value)
71
+ value.gsub!('\\\'', '\'')
72
+ value.gsub!('\\"', '"')
73
+ value = convert_placeholders_from_android_to_twine(value)
74
+ value.gsub!('\@', '@')
75
+ value.gsub!(/(\\u0020)*|(\\u0020)*\z/) { |spaces| ' ' * (spaces.length / 6) }
76
+ super(section, key, lang, value)
77
+ end
78
+
79
+ def read(io, lang)
80
+ document = REXML::Document.new io, :compress_whitespace => %w{ string }
81
+
82
+ comment = nil
83
+ document.root.children.each do |child|
84
+ if child.is_a? REXML::Comment
85
+ content = child.string.strip
86
+ comment = content if content.length > 0 and not content.start_with?("SECTION:")
87
+
88
+ elsif child.is_a? REXML::Element
89
+
90
+ if child.attributes['translatable']
91
+ next
92
+ end
93
+
94
+ section = nil
95
+ if child.name == 'plurals'
96
+ key = child.attributes['name']
97
+
98
+ if !section_exists(key)
99
+ section = TwineSection.new(key)
100
+ @twine_file.sections.insert(@twine_file.sections.size - 1, section)
101
+ else
102
+ section = get_section(key)
103
+ end
104
+
105
+ child.each do |item|
106
+ if item.is_a? REXML::Element
107
+ plural_key = key + '__' + item.attributes['quantity']
108
+ set_translation_for_key(section, plural_key, lang, item.text)
109
+ end
110
+ end
111
+ elsif child.name == 'string'
112
+
113
+ if !section_exists('Uncategorized')
114
+ section = TwineSection.new('Uncategorized')
115
+ @twine_file.sections.insert(0, section)
116
+ else
117
+ section = get_section('Uncategorized')
118
+ end
119
+
120
+ key = child.attributes['name']
121
+
122
+ set_translation_for_key(section, key, lang, child.text)
123
+ set_comment_for_key(key, comment) if comment
124
+
125
+ comment = nil
126
+ end
127
+ end
128
+ end
129
+ end
130
+
131
+ def format_header(lang)
132
+ "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!-- Android Strings File -->\n<!-- Generated by Twine #{Twine::VERSION} -->\n<!-- Language: #{lang} -->"
133
+ end
134
+
135
+ def format_sections(twine_file, lang)
136
+ result = '<resources>'
137
+
138
+ result += super + "\n"
139
+
140
+ result += "</resources>\n"
141
+ end
142
+
143
+ def format_section(section, lang)
144
+ definitions = section.definitions.select { |definition| should_include_definition(definition, lang) }
145
+ return if definitions.empty?
146
+
147
+ result = ""
148
+
149
+ if section.name && section.name.length > 0
150
+ section_header = format_section_header(section)
151
+ result += "\n#{section_header}" if section_header
152
+
153
+ # DEAL WITH PLURALS HERE
154
+ if section.is_uncategorized
155
+ definitions.map! { |definition| format_definition(definition, lang) }
156
+ definitions.compact! # remove nil definitions
157
+ definitions.map! { |definition| "\n#{definition}" } # prepend newline
158
+ result += definitions.join
159
+ else
160
+ result += plurals_start_key_value_pattern % { key:section.name }
161
+
162
+ definitions.map! { |definition| format_plural(definition, lang) }
163
+ definitions.compact! # remove nil definitions
164
+ definitions.map! { |definition| "\n#{definition}" } # prepend newline
165
+ result += definitions.join
166
+
167
+ result += plurals_end_key_value_pattern
168
+ end
169
+ end
170
+ end
171
+
172
+ def format_section_header(section)
173
+ " <!-- SECTION: #{section.name} -->"
174
+ end
175
+
176
+ def format_comment(definition, lang)
177
+ " <!-- #{definition.comment.gsub('--', '—')} -->\n" if definition.comment
178
+ end
179
+
180
+ def key_value_pattern
181
+ " <string name=\"%{key}\">%{value}</string>"
182
+ end
183
+
184
+ def format_plural(definition, lang)
185
+ [format_comment(definition, lang), format_key_value_plural_item(definition, lang)].compact.join
186
+ end
187
+
188
+ def format_key_value_plural_item(definition, lang)
189
+ value = definition.translation_for_lang(lang)
190
+ plurals_item_key_value_pattern(format_key(definition.key.dup), format_value(value.dup))
191
+ end
192
+
193
+ def plurals_start_key_value_pattern
194
+ "\n <plurals name=\"%{key}\">"
195
+ end
196
+
197
+ def plurals_item_key_value_pattern(key, value)
198
+ partitions = key.rpartition(/.__/)
199
+ " <item quantity=\"" + partitions[partitions.length - 1] + "\">" + value + "</item>"
200
+ end
201
+
202
+ def plurals_end_key_value_pattern
203
+ "\n </plurals>"
204
+ end
205
+
206
+ def escape_value(value)
207
+ # escape double and single quotes, & signs and tags
208
+ value = escape_quotes(value)
209
+ value.gsub!("'", "\\\\'")
210
+ value.gsub!(/&/, '&amp;')
211
+ value.gsub!('<', '&lt;')
212
+
213
+ # escape non resource identifier @ signs (http://developer.android.com/guide/topics/resources/accessing-resources.html#ResourcesFromXml)
214
+ resource_identifier_regex = /@(?!([a-z\.]+:)?[a-z+]+\/[a-zA-Z_]+)/ # @[<package_name>:]<resource_type>/<resource_name>
215
+ value.gsub(resource_identifier_regex, '\@')
216
+
217
+ value.gsub('strong>', 'b>')
218
+ end
219
+
220
+ # see http://developer.android.com/guide/topics/resources/string-resource.html#FormattingAndStyling
221
+ # however unescaped HTML markup like in "Welcome to <b>Android</b>!" is stripped when retrieved with getString() (http://stackoverflow.com/questions/9891996/)
222
+ def format_value(value)
223
+ value = value.dup
224
+
225
+ # capture xliff tags and replace them with a placeholder
226
+ xliff_tags = []
227
+ value.gsub! /<xliff:g.+?<\/xliff:g>/ do
228
+ xliff_tags << $&
229
+ 'TWINE_XLIFF_TAG_PLACEHOLDER'
230
+ end
231
+
232
+ # escape everything outside xliff tags
233
+ value = escape_value(value)
234
+
235
+ # put xliff tags back into place
236
+ xliff_tags.each do |xliff_tag|
237
+ # escape content of xliff tags
238
+ xliff_tag.gsub! /(<xliff:g.*?>)(.*)(<\/xliff:g>)/ do "#{$1}#{escape_value($2)}#{$3}" end
239
+ value.sub! 'TWINE_XLIFF_TAG_PLACEHOLDER', xliff_tag
240
+ end
241
+
242
+ # convert placeholders (e.g. %@ -> %s)
243
+ value = convert_placeholders_from_twine_to_android(value)
244
+
245
+ # replace beginning and end spaces with \u0020. Otherwise Android strips them.
246
+ value.gsub(/\A *| *\z/) { |spaces| '\u0020' * spaces.length }
247
+ value.gsub('%#s', '%#@')
248
+ end
249
+
250
+ end
251
+ end
252
+ end
253
+
254
+ Twine::Formatters.formatters << Twine::Formatters::Android.new