translatomatic 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +5 -5
  2. data/.gitattributes +1 -0
  3. data/.gitignore +15 -12
  4. data/.rspec +3 -3
  5. data/.travis.yml +32 -50
  6. data/CODE_OF_CONDUCT.md +74 -74
  7. data/Gemfile +29 -5
  8. data/Guardfile +48 -0
  9. data/LICENSE.txt +21 -21
  10. data/README.de.md +92 -0
  11. data/README.es.md +92 -0
  12. data/README.fr.md +92 -0
  13. data/README.it.md +92 -0
  14. data/README.ja.md +92 -0
  15. data/README.md +96 -74
  16. data/Rakefile +6 -6
  17. data/bin/setup +8 -8
  18. data/bin/translatomatic +6 -6
  19. data/bin/travis +26 -0
  20. data/db/database.yml +9 -9
  21. data/db/migrate/201712170000_initial.rb +24 -23
  22. data/lib/translatomatic/cli.rb +204 -80
  23. data/lib/translatomatic/config.rb +12 -26
  24. data/lib/translatomatic/converter.rb +206 -142
  25. data/lib/translatomatic/converter_stats.rb +27 -27
  26. data/lib/translatomatic/database.rb +139 -99
  27. data/lib/translatomatic/escaped_unicode.rb +90 -90
  28. data/lib/translatomatic/extractor/base.rb +14 -0
  29. data/lib/translatomatic/extractor/ruby.rb +5 -0
  30. data/lib/translatomatic/extractor.rb +4 -0
  31. data/lib/translatomatic/http_request.rb +133 -0
  32. data/lib/translatomatic/locale.rb +52 -0
  33. data/lib/translatomatic/logger.rb +28 -0
  34. data/lib/translatomatic/model/locale.rb +21 -22
  35. data/lib/translatomatic/model/text.rb +17 -13
  36. data/lib/translatomatic/model.rb +4 -4
  37. data/lib/translatomatic/option.rb +24 -24
  38. data/lib/translatomatic/progress_updater.rb +15 -0
  39. data/lib/translatomatic/resource_file/base.rb +169 -137
  40. data/lib/translatomatic/resource_file/html.rb +46 -28
  41. data/lib/translatomatic/resource_file/markdown.rb +54 -0
  42. data/lib/translatomatic/resource_file/plist.rb +30 -29
  43. data/lib/translatomatic/resource_file/properties.rb +72 -60
  44. data/lib/translatomatic/resource_file/resw.rb +30 -0
  45. data/lib/translatomatic/resource_file/text.rb +29 -28
  46. data/lib/translatomatic/resource_file/xcode_strings.rb +71 -65
  47. data/lib/translatomatic/resource_file/xml.rb +79 -59
  48. data/lib/translatomatic/resource_file/yaml.rb +82 -80
  49. data/lib/translatomatic/resource_file.rb +76 -74
  50. data/lib/translatomatic/string.rb +160 -0
  51. data/lib/translatomatic/tmx/document.rb +100 -0
  52. data/lib/translatomatic/tmx/translation_unit.rb +19 -0
  53. data/lib/translatomatic/tmx.rb +4 -0
  54. data/lib/translatomatic/translation_result.rb +75 -57
  55. data/lib/translatomatic/translator/base.rb +83 -47
  56. data/lib/translatomatic/translator/frengly.rb +57 -64
  57. data/lib/translatomatic/translator/google.rb +31 -30
  58. data/lib/translatomatic/translator/microsoft.rb +33 -32
  59. data/lib/translatomatic/translator/my_memory.rb +64 -55
  60. data/lib/translatomatic/translator/yandex.rb +39 -37
  61. data/lib/translatomatic/translator.rb +63 -63
  62. data/lib/translatomatic/util.rb +15 -24
  63. data/lib/translatomatic/version.rb +4 -3
  64. data/lib/translatomatic.rb +32 -27
  65. data/translatomatic.gemspec +43 -45
  66. metadata +52 -18
  67. data/Gemfile.lock +0 -137
@@ -1,137 +1,169 @@
1
- # @abstract Subclasses implement different types of resource files
2
- class Translatomatic::ResourceFile::Base
3
-
4
- attr_accessor :locale
5
- attr_accessor :path
6
-
7
- # @return [Hash<String,String>] key -> value properties
8
- attr_reader :properties
9
-
10
- # Create a new resource file.
11
- # If locale is unspecified, attempts to determine the locale of the file
12
- # automatically, and if that fails, uses the default locale.
13
- # @param [String] path Path to the file
14
- # @param [String] locale Locale of the file contents
15
- # @return [Translatomatic::ResourceFile::Base] the resource file.
16
- def initialize(path, locale = nil)
17
- @path = path.kind_of?(Pathname) ? path : Pathname.new(path)
18
- @locale = locale || detect_locale || parse_locale(I18n.default_locale)
19
- raise "unable to determine locale" unless @locale && @locale.language
20
- @valid = false
21
- @properties = {}
22
- end
23
-
24
- def format
25
- self.class.name.demodulize.downcase.to_sym
26
- end
27
-
28
- # Create a path for the current resource file with a given locale
29
- # @param [String] locale for the path
30
- # @return [Pathname] The path of this resource file modified for the given locale
31
- def locale_path(locale)
32
- basename = path.sub_ext('').basename.to_s
33
-
34
- extlist = extension_list
35
- if extlist.length >= 2 && loc_idx = find_locale(extlist)
36
- extlist[loc_idx] = locale.to_s
37
- elsif valid_locale?(basename)
38
- path.dirname + (locale.to_s + path.extname)
39
- else
40
- deunderscored = basename.sub(/_.*?$/, '')
41
- filename = deunderscored + "_" + locale.to_s + path.extname
42
- path.dirname + filename
43
- end
44
- end
45
-
46
- # Set all properties
47
- # @param [Hash<String,String>] properties New properties
48
- def properties=(properties)
49
- # use set rather that set @properties directly as subclasses override set()
50
- properties.each do |key, value|
51
- set(key, value)
52
- end
53
- end
54
-
55
- # Get the value of a property
56
- # @param [String] name The name of the property
57
- # @return [String] The value of the property
58
- def get(name)
59
- @properties[name]
60
- end
61
-
62
- # Set a property
63
- # @param [String] key The name of the property
64
- # @param [String] value The new value of the property
65
- # @return [String] The new value of the property
66
- def set(name, value)
67
- @properties[name] = value
68
- end
69
-
70
- # Test if the current resource file is valid
71
- # @return true if the current file is valid
72
- def valid?
73
- @valid
74
- end
75
-
76
- # Save the resource file.
77
- # @param [Pathname] target The destination path
78
- # @return [void]
79
- def save(target = path)
80
- raise "save(path) must be implemented by subclass"
81
- end
82
-
83
- # @return [String] String representation of this file
84
- def to_s
85
- "#{path.basename.to_s} (#{locale})"
86
- end
87
-
88
- private
89
-
90
- include Translatomatic::Util
91
-
92
- # detect locale from filename
93
- def detect_locale
94
- tag = nil
95
- basename = path.sub_ext('').basename.to_s
96
- directory = path.dirname.basename.to_s
97
- extlist = extension_list
98
-
99
- if basename.match(/_([-\w]{2,})$/i)
100
- # locale after underscore in filename
101
- tag = $1
102
- elsif directory.match(/^([-\w]+)\.lproj$/)
103
- # xcode localized strings
104
- tag = $1
105
- elsif extlist.length >= 2 && loc_idx = find_locale(extlist)
106
- # multiple parts to extension, e.g. index.html.en
107
- tag = extlist[loc_idx]
108
- elsif valid_locale?(basename)
109
- # try to match on entire basename
110
- # (support for rails en.yml)
111
- tag = basename
112
- end
113
-
114
- tag ? parse_locale(tag, true) : nil
115
- end
116
-
117
- # test if the list of strings contains a valid locale
118
- # return the index to the locale, or nil if no locales found
119
- def find_locale(list)
120
- list.find_index { |i| valid_locale?(i) }
121
- end
122
-
123
- # ext_sub() only removes the last extension
124
- def strip_extensions
125
- filename = path.basename.to_s
126
- filename.sub!(/\..*$/, '')
127
- path.parent + filename
128
- end
129
-
130
- # for index.html.de, returns ['html', 'de']
131
- def extension_list
132
- filename = path.basename.to_s
133
- idx = filename.index('.')
134
- idx && idx < filename.length - 1 ? filename[idx + 1..-1].split('.') : []
135
- end
136
-
137
- end
1
+ # @abstract Subclasses implement different types of resource files
2
+ class Translatomatic::ResourceFile::Base
3
+
4
+ attr_accessor :locale
5
+ attr_accessor :path
6
+
7
+ # @return [Hash<String,String>] key -> value properties
8
+ attr_reader :properties
9
+
10
+ # @return [Array<String>] File extensions supported by this resource file
11
+ def self.extensions
12
+ raise "extensions must be implemented by subclass"
13
+ end
14
+
15
+ # Create a new resource file.
16
+ # If locale is unspecified, attempts to determine the locale of the file
17
+ # automatically, and if that fails, uses the default locale.
18
+ # @param [String] path Path to the file
19
+ # @param [String] locale Locale of the file contents
20
+ # @return [Translatomatic::ResourceFile::Base] the resource file.
21
+ def initialize(path, locale = nil)
22
+ @path = path.kind_of?(Pathname) ? path : Pathname.new(path)
23
+ @locale = locale || detect_locale || Translatomatic::Locale.default
24
+ raise "unable to determine locale" unless @locale && @locale.language
25
+ @valid = false
26
+ @properties = {}
27
+ end
28
+
29
+ # @return [String] The format of this resource file, e.g. "Properties"
30
+ def format
31
+ self.class.name.demodulize.downcase.to_sym
32
+ end
33
+
34
+ # Create a path for the current resource file with a given locale
35
+ # @param [String] locale for the path
36
+ # @return [Pathname] The path of this resource file modified for the given locale
37
+ def locale_path(locale)
38
+ basename = path.sub_ext('').basename.to_s
39
+
40
+ extlist = extension_list
41
+ if extlist.length >= 2 && loc_idx = find_locale(extlist)
42
+ # extension(s) contains locale, replace it
43
+ extlist[loc_idx] = locale.to_s
44
+ elsif valid_locale?(basename)
45
+ # basename is a locale name, replace it
46
+ path.dirname + (locale.to_s + path.extname)
47
+ else
48
+ # remove any underscore and trailing text from basename
49
+ deunderscored = basename.sub(/_.*?$/, '')
50
+ # add _locale.ext
51
+ filename = deunderscored + "_" + locale.to_s + path.extname
52
+ path.dirname + filename
53
+ end
54
+ end
55
+
56
+ # Set all properties
57
+ # @param [Hash<String,String>] properties New properties
58
+ def properties=(properties)
59
+ # use set rather that set @properties directly as subclasses override set()
60
+ properties.each do |key, value|
61
+ set(key, value)
62
+ end
63
+ end
64
+
65
+ # Get the value of a property
66
+ # @param [String] name The name of the property
67
+ # @return [String] The value of the property
68
+ def get(name)
69
+ @properties[name]
70
+ end
71
+
72
+ # Set a property
73
+ # @param [String] key The name of the property
74
+ # @param [String] value The new value of the property
75
+ # @return [String] The new value of the property
76
+ def set(name, value)
77
+ @properties[name] = value
78
+ end
79
+
80
+ # Test if the current resource file is valid
81
+ # @return true if the current file is valid
82
+ def valid?
83
+ @valid
84
+ end
85
+
86
+ # Save the resource file.
87
+ # @param [Pathname] target The destination path
88
+ # @param [Hash<Symbol, Object>] options Output format options
89
+ # @return [void]
90
+ def save(target = path, options = {})
91
+ raise "save(path) must be implemented by subclass"
92
+ end
93
+
94
+ # @return [String] String representation of this file
95
+ def to_s
96
+ "#{path.basename.to_s} (#{locale})"
97
+ end
98
+
99
+ def sentences
100
+ sentences = []
101
+ properties.values.each do |value|
102
+ string = Translatomatic::String.new(value, locale)
103
+ sentences += string.sentences
104
+ end
105
+ sentences
106
+ end
107
+
108
+ private
109
+
110
+ include Translatomatic::Util
111
+
112
+ def created_by
113
+ date = DateTime.now.strftime("%Y-%m-%d %H:%M")
114
+ "Created by Translatomatic #{Translatomatic::VERSION} #{date}"
115
+ end
116
+
117
+ # detect locale from filename
118
+ def detect_locale
119
+ tag = nil
120
+ basename = path.sub_ext('').basename.to_s
121
+ directory = path.dirname.basename.to_s
122
+ extlist = extension_list
123
+
124
+ if basename.match(/_([-\w]{2,})$/i)
125
+ # locale after underscore in filename
126
+ tag = $1
127
+ elsif directory.match(/^([-\w]+)\.lproj$/)
128
+ # xcode localized strings
129
+ tag = $1
130
+ elsif extlist.length >= 2 && loc_idx = find_locale(extlist)
131
+ # multiple parts to extension, e.g. index.html.en
132
+ tag = extlist[loc_idx]
133
+ elsif valid_locale?(basename)
134
+ # try to match on entire basename
135
+ # (support for rails en.yml)
136
+ tag = basename
137
+ elsif valid_locale?(path.parent.basename)
138
+ # try to match on parent directory, e.g. strings/en-US/text.resw
139
+ tag = path.parent.basename
140
+ end
141
+
142
+ tag ? Translatomatic::Locale.parse(tag, true) : nil
143
+ end
144
+
145
+ def valid_locale?(tag)
146
+ Translatomatic::Locale.new(tag).valid?
147
+ end
148
+
149
+ # test if the list of strings contains a valid locale
150
+ # return the index to the locale, or nil if no locales found
151
+ def find_locale(list)
152
+ list.find_index { |i| valid_locale?(i) }
153
+ end
154
+
155
+ # ext_sub() only removes the last extension
156
+ def strip_extensions
157
+ filename = path.basename.to_s
158
+ filename.sub!(/\..*$/, '')
159
+ path.parent + filename
160
+ end
161
+
162
+ # for index.html.de, returns ['html', 'de']
163
+ def extension_list
164
+ filename = path.basename.to_s
165
+ idx = filename.index('.')
166
+ idx && idx < filename.length - 1 ? filename[idx + 1..-1].split('.') : []
167
+ end
168
+
169
+ end
@@ -1,33 +1,51 @@
1
- module Translatomatic::ResourceFile
2
- class HTML < XML
3
-
4
- def self.extensions
5
- %w{html htm shtml}
6
- end
7
-
8
- # (see Translatomatic::ResourceFile::Base#locale_path)
9
- def locale_path(locale)
10
- extlist = extension_list
11
- if extlist.length >= 2 && loc_idx = find_locale(extlist)
12
- # part of the extension is the locale
13
- # replace that part with the new locale
14
- extlist[loc_idx] = locale.to_s
15
- new_extension = extlist.join(".")
16
- return strip_extensions.sub_ext("." + new_extension)
17
- else
18
- # add locale extension
1
+ module Translatomatic::ResourceFile
2
+ class HTML < XML
3
+
4
+ # (see Translatomatic::ResourceFile::Base.extensions)
5
+ def self.extensions
6
+ %w{html htm shtml}
7
+ end
8
+
9
+ # (see Translatomatic::ResourceFile::Base#locale_path)
10
+ def locale_path(locale)
11
+ extlist = extension_list
12
+ if extlist.length >= 2 && loc_idx = find_locale(extlist)
13
+ # part of the extension is the locale
14
+ # replace that part with the new locale
15
+ extlist[loc_idx] = locale.to_s
16
+ new_extension = extlist.join(".")
17
+ return strip_extensions.sub_ext("." + new_extension)
18
+ else
19
+ # add locale extension
19
20
  ext = path.extname
20
- path.sub_ext("#{ext}." + locale.to_s)
21
- end
21
+ # TODO: need configurable order for locale & ext here?
22
+ #path.sub_ext("#{ext}." + locale.to_s)
23
+ path.sub_ext("." + locale.to_s + ext)
24
+ end
25
+ end
26
+
27
+ # (see Translatomatic::ResourceFile::Base#save)
28
+ def save(target = path, options = {})
29
+ if @doc
30
+ add_created_by unless options[:no_created_by]
31
+ target.write(@doc.to_html)
32
+ end
33
+ end
34
+
35
+ private
22
36
 
23
- # fall back to base functionality
24
- #super(locale)
37
+ def text_nodes_xpath
38
+ '//*[not(self::code)]/text()'
25
39
  end
26
40
 
27
- # (see Translatomatic::ResourceFile::Base#save(path))
28
- def save(target = path)
29
- target.write(@doc.to_html) if @doc
41
+ def add_created_by
42
+ @created_by ||= @doc.root.add_previous_sibling(comment(created_by))
30
43
  end
31
-
32
- end
33
- end
44
+
45
+ def read_doc(path)
46
+ Nokogiri::HTML(path.open) do |config|
47
+ config.noblanks
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,54 @@
1
+ require 'kramdown'
2
+ require 'reverse_markdown'
3
+
4
+ module Translatomatic::ResourceFile
5
+ class Markdown < HTML
6
+
7
+ # (see Translatomatic::ResourceFile::Base.extensions)
8
+ def self.extensions
9
+ %w{md}
10
+ end
11
+
12
+ # (see Translatomatic::ResourceFile::Base#save)
13
+ def save(target = path, options = {})
14
+ if @doc
15
+ begin
16
+ add_created_by unless options[:no_created_by]
17
+ html = @doc.to_html
18
+ # convert html back to markdown
19
+ markdown = ReverseMarkdown.convert(html, unknown_tags: :bypass)
20
+ target.write(markdown.chomp)
21
+ rescue Exception => e
22
+ puts "error: #{e.message}"
23
+ end
24
+ end
25
+ end
26
+
27
+ private
28
+
29
+ def add_created_by
30
+ @created_by ||= begin
31
+ body = @doc.at('body')
32
+ body.add_child("<p><i>#{created_by}</i></p>")
33
+ end
34
+ end
35
+
36
+ def read(path)
37
+ begin
38
+ # read markdown and convert to html
39
+ markdown = path.read
40
+ html = Kramdown::Document.new(markdown).to_html
41
+ # parse html with nokogiri
42
+ @doc = Nokogiri::HTML(html) do |config|
43
+ config.noblanks
44
+ end
45
+ init_nodemap(@doc)
46
+ rescue Exception => e
47
+ log.error(e.message)
48
+ @valid = false
49
+ {}
50
+ end
51
+ end
52
+
53
+ end # class
54
+ end # module
@@ -1,29 +1,30 @@
1
- module Translatomatic::ResourceFile
2
- class Plist < XML
3
-
4
- def self.extensions
5
- %w{plist}
6
- end
7
-
8
- # (see Translatomatic::ResourceFile::Base#locale_path)
9
- # @note localization files in XCode use the following file name
10
- # convention: Project/locale.lproj/filename
11
- # @todo refactor this and xcode_strings.rb to use the same code
12
- def locale_path(locale)
13
- if path.to_s.match(/\/([-\w]+).lproj\/.+.plist$/)
14
- # xcode style
15
- filename = path.basename
16
- path.parent.parent + (locale.to_s + ".lproj") + filename
17
- else
18
- super(locale)
19
- end
20
- end
21
-
22
- private
23
-
24
- def text_nodes_xpath
25
- '//*[not(self::key)]/text()'
26
- end
27
-
28
- end # class
29
- end # module
1
+ module Translatomatic::ResourceFile
2
+ class Plist < XML
3
+
4
+ # (see Translatomatic::ResourceFile::Base.extensions)
5
+ def self.extensions
6
+ %w{plist}
7
+ end
8
+
9
+ # (see Translatomatic::ResourceFile::Base#locale_path)
10
+ # @note localization files in XCode use the following file name
11
+ # convention: Project/locale.lproj/filename
12
+ # @todo refactor this and xcode_strings.rb to use the same code
13
+ def locale_path(locale)
14
+ if path.to_s.match(/\/([-\w]+).lproj\/.+.plist$/)
15
+ # xcode style
16
+ filename = path.basename
17
+ path.parent.parent + (locale.to_s + ".lproj") + filename
18
+ else
19
+ super(locale)
20
+ end
21
+ end
22
+
23
+ private
24
+
25
+ def text_nodes_xpath
26
+ '//*[not(self::key)]/text()'
27
+ end
28
+
29
+ end # class
30
+ end # module
@@ -1,60 +1,72 @@
1
- module Translatomatic::ResourceFile
2
- class Properties < Base
3
-
4
- def self.extensions
5
- %w{properties}
6
- end
7
-
8
- # (see Translatomatic::ResourceFile::Base#initialize)
9
- def initialize(path, locale = nil)
10
- super(path, locale)
11
- @valid = true
12
- @properties = @path.exist? ? read(@path) : {}
13
- end
14
-
15
- # (see Translatomatic::ResourceFile::Base#save(target))
16
- def save(target = path)
17
- out = ""
18
- properties.each do |key, value|
19
- # TODO: maintain original line ending format?
20
- value = value.gsub("\n", "\\n") # convert newlines to \n
21
- out += "#{key} = #{value}\n"
22
- end
23
- # escape unicode characters
24
- out = Translatomatic::EscapedUnicode.escape(out)
25
- target.write(out)
26
- end
27
-
28
- private
29
-
30
- # parse key = value property file
31
- def read(path)
32
- contents = path.read
33
- # convert escaped unicode characters into unicode
34
- contents = Translatomatic::EscapedUnicode.unescape(contents)
35
- result = {}
36
- contents.gsub!(/\\\s*\n\s*/m, '') # put multi line strings on one line
37
- lines = contents.split("\n")
38
-
39
- lines.each do |line|
40
- line.strip!
41
- next if line.length == 0
42
- equal_idx = line.index("=")
43
-
44
- if line[0] == ?! || line[0] == ?#
45
- # comment
46
- # TODO: translate comments or keep originals?
47
- next
48
- elsif equal_idx.nil?
49
- @valid = false
50
- return {}
51
- end
52
- name, value = line.split(/\s*=\s*/, 2)
53
- value = value.gsub("\\n", "\n") # convert \n to newlines
54
- result[name] = value
55
- end
56
- result
57
- end
58
-
59
- end
60
- end
1
+ require 'date'
2
+
3
+ module Translatomatic::ResourceFile
4
+ class Properties < Base
5
+
6
+ # (see Translatomatic::ResourceFile::Base.extensions)
7
+ def self.extensions
8
+ %w{properties}
9
+ end
10
+
11
+ # (see Translatomatic::ResourceFile::Base#initialize)
12
+ def initialize(path, locale = nil)
13
+ super(path, locale)
14
+ @valid = true
15
+ @properties = @path.exist? ? read(@path) : {}
16
+ end
17
+
18
+ # (see Translatomatic::ResourceFile::Base#save)
19
+ def save(target = path, options = {})
20
+ out = ""
21
+ out += add_created_by unless options[:no_created_by]
22
+ properties.each do |key, value|
23
+ # TODO: maintain original line ending format?
24
+ value = value.gsub("\n", "\\n") # convert newlines to \n
25
+ out += "#{key} = #{value}\n"
26
+ end
27
+ # escape unicode characters
28
+ out = Translatomatic::EscapedUnicode.escape(out)
29
+ target.write(out)
30
+ end
31
+
32
+ private
33
+
34
+ def add_created_by
35
+ comment(created_by)
36
+ end
37
+
38
+ def comment(text)
39
+ "# #{text}\n"
40
+ end
41
+
42
+ # parse key = value property file
43
+ def read(path)
44
+ contents = path.read
45
+ # convert escaped unicode characters into unicode
46
+ contents = Translatomatic::EscapedUnicode.unescape(contents)
47
+ result = {}
48
+ contents.gsub!(/\\\s*\n\s*/m, '') # put multi line strings on one line
49
+ lines = contents.split("\n")
50
+
51
+ lines.each do |line|
52
+ line.strip!
53
+ next if line.length == 0
54
+ equal_idx = line.index("=")
55
+
56
+ if line[0] == ?! || line[0] == ?#
57
+ # comment
58
+ # TODO: translate comments or keep originals?
59
+ next
60
+ elsif equal_idx.nil?
61
+ @valid = false
62
+ return {}
63
+ end
64
+ name, value = line.split(/\s*=\s*/, 2)
65
+ value = value.gsub("\\n", "\n") # convert \n to newlines
66
+ result[name] = value
67
+ end
68
+ result
69
+ end
70
+
71
+ end
72
+ end