yury-twine 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +2 -0
- data/LICENSE +30 -0
- data/README.md +230 -0
- data/bin/twine +7 -0
- data/lib/twine.rb +36 -0
- data/lib/twine/cli.rb +200 -0
- data/lib/twine/encoding.rb +22 -0
- data/lib/twine/formatters.rb +20 -0
- data/lib/twine/formatters/abstract.rb +187 -0
- data/lib/twine/formatters/android.rb +254 -0
- data/lib/twine/formatters/apple.rb +328 -0
- data/lib/twine/output_processor.rb +57 -0
- data/lib/twine/placeholders.rb +54 -0
- data/lib/twine/plugin.rb +62 -0
- data/lib/twine/runner.rb +332 -0
- data/lib/twine/twine_file.rb +266 -0
- data/lib/twine/version.rb +3 -0
- data/test/command_test.rb +14 -0
- data/test/fixtures/consume_loc_drop.zip +0 -0
- data/test/fixtures/enc_utf16be.dummy +0 -0
- data/test/fixtures/enc_utf16be_bom.dummy +0 -0
- data/test/fixtures/enc_utf16le.dummy +0 -0
- data/test/fixtures/enc_utf16le_bom.dummy +0 -0
- data/test/fixtures/enc_utf8.dummy +2 -0
- data/test/fixtures/formatter_android.xml +15 -0
- data/test/fixtures/formatter_apple.strings +20 -0
- data/test/fixtures/formatter_django.po +30 -0
- data/test/fixtures/formatter_flash.properties +15 -0
- data/test/fixtures/formatter_gettext.po +26 -0
- data/test/fixtures/formatter_jquery.json +7 -0
- data/test/fixtures/formatter_tizen.xml +15 -0
- data/test/fixtures/gettext_multiline.po +10 -0
- data/test/fixtures/twine_accent_values.txt +13 -0
- data/test/test_abstract_formatter.rb +165 -0
- data/test/test_cli.rb +304 -0
- data/test/test_consume_loc_drop.rb +27 -0
- data/test/test_consume_localization_file.rb +119 -0
- data/test/test_formatters.rb +363 -0
- data/test/test_generate_all_localization_files.rb +102 -0
- data/test/test_generate_loc_drop.rb +80 -0
- data/test/test_generate_localization_file.rb +91 -0
- data/test/test_output_processor.rb +85 -0
- data/test/test_placeholders.rb +84 -0
- data/test/test_twine_definition.rb +111 -0
- data/test/test_twine_file.rb +58 -0
- data/test/test_validate_twine_file.rb +61 -0
- data/test/twine_file_dsl.rb +46 -0
- data/test/twine_test.rb +48 -0
- metadata +179 -0
@@ -0,0 +1,22 @@
|
|
1
|
+
module Twine
|
2
|
+
module Encoding
|
3
|
+
|
4
|
+
def self.bom(path)
|
5
|
+
first_bytes = IO.binread(path, 2)
|
6
|
+
return nil unless first_bytes
|
7
|
+
first_bytes = first_bytes.codepoints.map.to_a
|
8
|
+
return 'UTF-16BE' if first_bytes == [0xFE, 0xFF]
|
9
|
+
return 'UTF-16LE' if first_bytes == [0xFF, 0xFE]
|
10
|
+
rescue EOFError
|
11
|
+
return nil
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.has_bom?(path)
|
15
|
+
!bom(path).nil?
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.encoding_for_path(path)
|
19
|
+
bom(path) || 'UTF-8'
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Twine
|
2
|
+
module Formatters
|
3
|
+
@formatters = []
|
4
|
+
|
5
|
+
class << self
|
6
|
+
attr_reader :formatters
|
7
|
+
|
8
|
+
###
|
9
|
+
# registers a new formatter
|
10
|
+
#
|
11
|
+
# formatter_class - the class of the formatter to register
|
12
|
+
#
|
13
|
+
# returns array of active formatters
|
14
|
+
#
|
15
|
+
def register_formatter formatter_class
|
16
|
+
@formatters << formatter_class.new
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,187 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
|
3
|
+
module Twine
|
4
|
+
module Formatters
|
5
|
+
class Abstract
|
6
|
+
attr_accessor :twine_file
|
7
|
+
attr_accessor :options
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@twine_file = TwineFile.new
|
11
|
+
@options = {}
|
12
|
+
end
|
13
|
+
|
14
|
+
def format_name
|
15
|
+
raise NotImplementedError.new("You must implement format_name in your formatter class.")
|
16
|
+
end
|
17
|
+
|
18
|
+
def extension
|
19
|
+
raise NotImplementedError.new("You must implement extension in your formatter class.")
|
20
|
+
end
|
21
|
+
|
22
|
+
def can_handle_directory?(path)
|
23
|
+
raise NotImplementedError.new("You must implement can_handle_directory? in your formatter class.")
|
24
|
+
end
|
25
|
+
|
26
|
+
def can_handle_file?(path)
|
27
|
+
raise NotImplementedError.new("You must implement can_handle_file? in your formatter class.")
|
28
|
+
end
|
29
|
+
|
30
|
+
def default_file_name
|
31
|
+
raise NotImplementedError.new("You must implement default_file_name in your formatter class.")
|
32
|
+
end
|
33
|
+
|
34
|
+
def set_translation_for_key(current_section, key, lang, value)
|
35
|
+
value = value.gsub("\n", "\\n")
|
36
|
+
|
37
|
+
if @twine_file.definitions_by_key.include?(key)
|
38
|
+
definition = @twine_file.definitions_by_key[key]
|
39
|
+
reference = @twine_file.definitions_by_key[definition.reference_key] if definition.reference_key
|
40
|
+
|
41
|
+
if !reference or value != reference.translations[lang]
|
42
|
+
definition.translations[lang] = value
|
43
|
+
end
|
44
|
+
elsif @options[:consume_all]
|
45
|
+
Twine::stderr.puts "Adding new definition '#{key}' to twine file."
|
46
|
+
current_definition = TwineDefinition.new(key)
|
47
|
+
current_section.definitions << current_definition
|
48
|
+
|
49
|
+
if @options[:tags] && @options[:tags].length > 0
|
50
|
+
current_definition.tags = @options[:tags]
|
51
|
+
end
|
52
|
+
|
53
|
+
@twine_file.definitions_by_key[key] = current_definition
|
54
|
+
@twine_file.definitions_by_key[key].translations[lang] = value
|
55
|
+
else
|
56
|
+
Twine::stderr.puts "Warning: '#{key}' not found in twine file."
|
57
|
+
end
|
58
|
+
if !@twine_file.language_codes.include?(lang)
|
59
|
+
@twine_file.add_language_code(lang)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def set_comment_for_key(key, comment)
|
64
|
+
return unless @options[:consume_comments]
|
65
|
+
|
66
|
+
if @twine_file.definitions_by_key.include?(key)
|
67
|
+
definition = @twine_file.definitions_by_key[key]
|
68
|
+
|
69
|
+
reference = @twine_file.definitions_by_key[definition.reference_key] if definition.reference_key
|
70
|
+
|
71
|
+
if !reference or comment != reference.raw_comment
|
72
|
+
definition.comment = comment
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def set_ios_comment_for_key(key, comment)
|
78
|
+
if @twine_file.definitions_by_key.include?(key)
|
79
|
+
definition = @twine_file.definitions_by_key[key]
|
80
|
+
|
81
|
+
reference = @twine_file.definitions_by_key[definition.reference_key] if definition.reference_key
|
82
|
+
|
83
|
+
if !reference or ios_comment != reference.raw_ios_comment
|
84
|
+
definition.ios_comment = comment
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def determine_language_given_path(path)
|
90
|
+
raise NotImplementedError.new("You must implement determine_language_given_path in your formatter class.")
|
91
|
+
end
|
92
|
+
|
93
|
+
def output_path_for_language(lang)
|
94
|
+
lang
|
95
|
+
end
|
96
|
+
|
97
|
+
def read(io, lang)
|
98
|
+
raise NotImplementedError.new("You must implement read in your formatter class.")
|
99
|
+
end
|
100
|
+
|
101
|
+
def format_file(lang)
|
102
|
+
output_processor = Processors::OutputProcessor.new(@twine_file, @options)
|
103
|
+
processed_twine_file = output_processor.process(lang)
|
104
|
+
|
105
|
+
return nil if processed_twine_file.definitions_by_key.empty?
|
106
|
+
|
107
|
+
header = format_header(lang)
|
108
|
+
result = ""
|
109
|
+
result += header + "\n" if header
|
110
|
+
result += format_sections(processed_twine_file, lang)
|
111
|
+
end
|
112
|
+
|
113
|
+
def format_header(lang)
|
114
|
+
end
|
115
|
+
|
116
|
+
def format_sections(twine_file, lang)
|
117
|
+
sections = twine_file.sections.map { |section| format_section(section, lang) }
|
118
|
+
sections.compact.join("\n")
|
119
|
+
end
|
120
|
+
|
121
|
+
def format_section_header(section)
|
122
|
+
end
|
123
|
+
|
124
|
+
def should_include_definition(definition, lang)
|
125
|
+
return !definition.translation_for_lang(lang).nil?
|
126
|
+
end
|
127
|
+
|
128
|
+
def format_section(section, lang)
|
129
|
+
definitions = section.definitions.select { |definition| should_include_definition(definition, lang) }
|
130
|
+
return if definitions.empty?
|
131
|
+
|
132
|
+
result = ""
|
133
|
+
|
134
|
+
if section.name && section.name.length > 0
|
135
|
+
section_header = format_section_header(section)
|
136
|
+
result += "\n#{section_header}" if section_header
|
137
|
+
end
|
138
|
+
|
139
|
+
definitions.map! { |definition| format_definition(definition, lang) }
|
140
|
+
definitions.compact! # remove nil definitions
|
141
|
+
definitions.map! { |definition| "\n#{definition}" } # prepend newline
|
142
|
+
result += definitions.join
|
143
|
+
end
|
144
|
+
|
145
|
+
def format_definition(definition, lang)
|
146
|
+
[format_comment(definition, lang), format_key_value(definition, lang)].compact.join
|
147
|
+
end
|
148
|
+
|
149
|
+
def format_comment(definition, lang)
|
150
|
+
end
|
151
|
+
|
152
|
+
def format_key_value(definition, lang)
|
153
|
+
value = definition.translation_for_lang(lang)
|
154
|
+
key_value_pattern % { key: format_key(definition.key.dup), value: format_value(value.dup) }
|
155
|
+
end
|
156
|
+
|
157
|
+
def key_value_pattern
|
158
|
+
raise NotImplementedError.new("You must implement key_value_pattern in your formatter class.")
|
159
|
+
end
|
160
|
+
|
161
|
+
def format_key(key)
|
162
|
+
key
|
163
|
+
end
|
164
|
+
|
165
|
+
def format_value(value)
|
166
|
+
value
|
167
|
+
end
|
168
|
+
|
169
|
+
def escape_quotes(text)
|
170
|
+
text.gsub('"', "\\\"")
|
171
|
+
end
|
172
|
+
|
173
|
+
def section_exists(section_name)
|
174
|
+
@twine_file.sections.find { |s| s.name == section_name }
|
175
|
+
end
|
176
|
+
|
177
|
+
def get_section(name)
|
178
|
+
@twine_file.sections.each do |s|
|
179
|
+
if s.name == name
|
180
|
+
return s
|
181
|
+
end
|
182
|
+
end
|
183
|
+
return nil
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
187
|
+
end
|
@@ -0,0 +1,254 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'cgi'
|
3
|
+
require 'rexml/document'
|
4
|
+
|
5
|
+
module Twine
|
6
|
+
module Formatters
|
7
|
+
class Android < Abstract
|
8
|
+
include Twine::Placeholders
|
9
|
+
|
10
|
+
LANG_MAPPINGS = Hash[
|
11
|
+
'zh-rCN' => 'zh-Hans',
|
12
|
+
'zh-rHK' => 'zh-Hant',
|
13
|
+
'en-rGB' => 'en-UK',
|
14
|
+
'in' => 'id',
|
15
|
+
'nb' => 'no'
|
16
|
+
# TODO: spanish
|
17
|
+
]
|
18
|
+
|
19
|
+
def format_name
|
20
|
+
'android'
|
21
|
+
end
|
22
|
+
|
23
|
+
def extension
|
24
|
+
'.xml'
|
25
|
+
end
|
26
|
+
|
27
|
+
def can_handle_directory?(path)
|
28
|
+
Dir.entries(path).any? { |item| /^values.*$/.match(item) }
|
29
|
+
end
|
30
|
+
|
31
|
+
def can_handle_file?(path)
|
32
|
+
path_arr = path.split(File::SEPARATOR)
|
33
|
+
return path_arr[path_arr.length - 1] == default_file_name
|
34
|
+
end
|
35
|
+
|
36
|
+
def default_file_name
|
37
|
+
return 'strings.xml'
|
38
|
+
end
|
39
|
+
|
40
|
+
def determine_language_given_path(path)
|
41
|
+
path_arr = path.split(File::SEPARATOR)
|
42
|
+
path_arr.each do |segment|
|
43
|
+
if segment == 'values'
|
44
|
+
return 'en'
|
45
|
+
else
|
46
|
+
# The language is defined by a two-letter ISO 639-1 language code, optionally followed by a two letter ISO 3166-1-alpha-2 region code (preceded by lowercase "r").
|
47
|
+
# see http://developer.android.com/guide/topics/resources/providing-resources.html#AlternativeResources
|
48
|
+
match = /^values-([a-z]{2}(-r[a-z]{2})?)$/i.match(segment)
|
49
|
+
if match
|
50
|
+
lang = match[1]
|
51
|
+
lang = LANG_MAPPINGS.fetch(lang, lang)
|
52
|
+
lang.sub!('-r', '-')
|
53
|
+
return lang
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
return
|
59
|
+
end
|
60
|
+
|
61
|
+
def output_path_for_language(lang)
|
62
|
+
if lang == 'en'
|
63
|
+
"values"
|
64
|
+
else
|
65
|
+
"values-" + (LANG_MAPPINGS.key(lang) || lang)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def set_translation_for_key(section, key, lang, value)
|
70
|
+
value = CGI.unescapeHTML(value)
|
71
|
+
value.gsub!('\\\'', '\'')
|
72
|
+
value.gsub!('\\"', '"')
|
73
|
+
value = convert_placeholders_from_android_to_twine(value)
|
74
|
+
value.gsub!('\@', '@')
|
75
|
+
value.gsub!(/(\\u0020)*|(\\u0020)*\z/) { |spaces| ' ' * (spaces.length / 6) }
|
76
|
+
super(section, key, lang, value)
|
77
|
+
end
|
78
|
+
|
79
|
+
def read(io, lang)
|
80
|
+
document = REXML::Document.new io, :compress_whitespace => %w{ string }
|
81
|
+
|
82
|
+
comment = nil
|
83
|
+
document.root.children.each do |child|
|
84
|
+
if child.is_a? REXML::Comment
|
85
|
+
content = child.string.strip
|
86
|
+
comment = content if content.length > 0 and not content.start_with?("SECTION:")
|
87
|
+
|
88
|
+
elsif child.is_a? REXML::Element
|
89
|
+
|
90
|
+
if child.attributes['translatable']
|
91
|
+
next
|
92
|
+
end
|
93
|
+
|
94
|
+
section = nil
|
95
|
+
if child.name == 'plurals'
|
96
|
+
key = child.attributes['name']
|
97
|
+
|
98
|
+
if !section_exists(key)
|
99
|
+
section = TwineSection.new(key)
|
100
|
+
@twine_file.sections.insert(@twine_file.sections.size - 1, section)
|
101
|
+
else
|
102
|
+
section = get_section(key)
|
103
|
+
end
|
104
|
+
|
105
|
+
child.each do |item|
|
106
|
+
if item.is_a? REXML::Element
|
107
|
+
plural_key = key + '__' + item.attributes['quantity']
|
108
|
+
set_translation_for_key(section, plural_key, lang, item.text)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
elsif child.name == 'string'
|
112
|
+
|
113
|
+
if !section_exists('Uncategorized')
|
114
|
+
section = TwineSection.new('Uncategorized')
|
115
|
+
@twine_file.sections.insert(0, section)
|
116
|
+
else
|
117
|
+
section = get_section('Uncategorized')
|
118
|
+
end
|
119
|
+
|
120
|
+
key = child.attributes['name']
|
121
|
+
|
122
|
+
set_translation_for_key(section, key, lang, child.text)
|
123
|
+
set_comment_for_key(key, comment) if comment
|
124
|
+
|
125
|
+
comment = nil
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def format_header(lang)
|
132
|
+
"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!-- Android Strings File -->\n<!-- Generated by Twine #{Twine::VERSION} -->\n<!-- Language: #{lang} -->"
|
133
|
+
end
|
134
|
+
|
135
|
+
def format_sections(twine_file, lang)
|
136
|
+
result = '<resources>'
|
137
|
+
|
138
|
+
result += super + "\n"
|
139
|
+
|
140
|
+
result += "</resources>\n"
|
141
|
+
end
|
142
|
+
|
143
|
+
def format_section(section, lang)
|
144
|
+
definitions = section.definitions.select { |definition| should_include_definition(definition, lang) }
|
145
|
+
return if definitions.empty?
|
146
|
+
|
147
|
+
result = ""
|
148
|
+
|
149
|
+
if section.name && section.name.length > 0
|
150
|
+
section_header = format_section_header(section)
|
151
|
+
result += "\n#{section_header}" if section_header
|
152
|
+
|
153
|
+
# DEAL WITH PLURALS HERE
|
154
|
+
if section.is_uncategorized
|
155
|
+
definitions.map! { |definition| format_definition(definition, lang) }
|
156
|
+
definitions.compact! # remove nil definitions
|
157
|
+
definitions.map! { |definition| "\n#{definition}" } # prepend newline
|
158
|
+
result += definitions.join
|
159
|
+
else
|
160
|
+
result += plurals_start_key_value_pattern % { key:section.name }
|
161
|
+
|
162
|
+
definitions.map! { |definition| format_plural(definition, lang) }
|
163
|
+
definitions.compact! # remove nil definitions
|
164
|
+
definitions.map! { |definition| "\n#{definition}" } # prepend newline
|
165
|
+
result += definitions.join
|
166
|
+
|
167
|
+
result += plurals_end_key_value_pattern
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
def format_section_header(section)
|
173
|
+
" <!-- SECTION: #{section.name} -->"
|
174
|
+
end
|
175
|
+
|
176
|
+
def format_comment(definition, lang)
|
177
|
+
" <!-- #{definition.comment.gsub('--', '—')} -->\n" if definition.comment
|
178
|
+
end
|
179
|
+
|
180
|
+
def key_value_pattern
|
181
|
+
" <string name=\"%{key}\">%{value}</string>"
|
182
|
+
end
|
183
|
+
|
184
|
+
def format_plural(definition, lang)
|
185
|
+
[format_comment(definition, lang), format_key_value_plural_item(definition, lang)].compact.join
|
186
|
+
end
|
187
|
+
|
188
|
+
def format_key_value_plural_item(definition, lang)
|
189
|
+
value = definition.translation_for_lang(lang)
|
190
|
+
plurals_item_key_value_pattern(format_key(definition.key.dup), format_value(value.dup))
|
191
|
+
end
|
192
|
+
|
193
|
+
def plurals_start_key_value_pattern
|
194
|
+
"\n <plurals name=\"%{key}\">"
|
195
|
+
end
|
196
|
+
|
197
|
+
def plurals_item_key_value_pattern(key, value)
|
198
|
+
partitions = key.rpartition(/.__/)
|
199
|
+
" <item quantity=\"" + partitions[partitions.length - 1] + "\">" + value + "</item>"
|
200
|
+
end
|
201
|
+
|
202
|
+
def plurals_end_key_value_pattern
|
203
|
+
"\n </plurals>"
|
204
|
+
end
|
205
|
+
|
206
|
+
def escape_value(value)
|
207
|
+
# escape double and single quotes, & signs and tags
|
208
|
+
value = escape_quotes(value)
|
209
|
+
value.gsub!("'", "\\\\'")
|
210
|
+
value.gsub!(/&/, '&')
|
211
|
+
value.gsub!('<', '<')
|
212
|
+
|
213
|
+
# escape non resource identifier @ signs (http://developer.android.com/guide/topics/resources/accessing-resources.html#ResourcesFromXml)
|
214
|
+
resource_identifier_regex = /@(?!([a-z\.]+:)?[a-z+]+\/[a-zA-Z_]+)/ # @[<package_name>:]<resource_type>/<resource_name>
|
215
|
+
value.gsub(resource_identifier_regex, '\@')
|
216
|
+
|
217
|
+
value.gsub('strong>', 'b>')
|
218
|
+
end
|
219
|
+
|
220
|
+
# see http://developer.android.com/guide/topics/resources/string-resource.html#FormattingAndStyling
|
221
|
+
# however unescaped HTML markup like in "Welcome to <b>Android</b>!" is stripped when retrieved with getString() (http://stackoverflow.com/questions/9891996/)
|
222
|
+
def format_value(value)
|
223
|
+
value = value.dup
|
224
|
+
|
225
|
+
# capture xliff tags and replace them with a placeholder
|
226
|
+
xliff_tags = []
|
227
|
+
value.gsub! /<xliff:g.+?<\/xliff:g>/ do
|
228
|
+
xliff_tags << $&
|
229
|
+
'TWINE_XLIFF_TAG_PLACEHOLDER'
|
230
|
+
end
|
231
|
+
|
232
|
+
# escape everything outside xliff tags
|
233
|
+
value = escape_value(value)
|
234
|
+
|
235
|
+
# put xliff tags back into place
|
236
|
+
xliff_tags.each do |xliff_tag|
|
237
|
+
# escape content of xliff tags
|
238
|
+
xliff_tag.gsub! /(<xliff:g.*?>)(.*)(<\/xliff:g>)/ do "#{$1}#{escape_value($2)}#{$3}" end
|
239
|
+
value.sub! 'TWINE_XLIFF_TAG_PLACEHOLDER', xliff_tag
|
240
|
+
end
|
241
|
+
|
242
|
+
# convert placeholders (e.g. %@ -> %s)
|
243
|
+
value = convert_placeholders_from_twine_to_android(value)
|
244
|
+
|
245
|
+
# replace beginning and end spaces with \u0020. Otherwise Android strips them.
|
246
|
+
value.gsub(/\A *| *\z/) { |spaces| '\u0020' * spaces.length }
|
247
|
+
value.gsub('%#s', '%#@')
|
248
|
+
end
|
249
|
+
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
Twine::Formatters.formatters << Twine::Formatters::Android.new
|