tmx_importer 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 879c5a494ffdf3380ca31ffc402dc031261db2bb
-  data.tar.gz: 09cd7465ac1b8cb93fc21edc411cba48ca20ae7d
+  metadata.gz: 9171f015c2f0e45c3772d64d0f98e53949abf89a
+  data.tar.gz: f227573b58faf7b215695d9a72ff12df051bd6e9
 SHA512:
-  metadata.gz: a1e9252071f93e9a8a65f811a931e7c0aef4f914b7709d28780be54f69f543508773ac2fe379d2c67587abcaa2ce074b41bb9e1eceb628e6f61c432c5c9f0d80
-  data.tar.gz: 7e8e76da0b91f2ba041e75e7d86d348d6ad1fe66f03506173e0535d6e0395f9e12a0bd04df9699e4141a931bc1da1aad74498af37c54e0fdd7bba1bc4a30c7c9
+  metadata.gz: b9dd9c281a0453d994244f87aeaf466c46aa98336a3bae7a598344df23bb01b66779d235f305533e421f04b1a6ca1a695745a28b7311abb18ed3563b724478a8
+  data.tar.gz: 8d61b7d51f8b6ed56c21b68639f52aaa3c3f9ed1e6ee158cf1ecfed7669c77cdafb65945e718d36228545d281e9b385ed416639e0413ecfee258ed2922ee43b1

data/lib/tmx_importer/version.rb

@@ -1,3 +1,3 @@
 module TmxImporter
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/tmx_importer.rb

@@ -3,9 +3,6 @@ require 'xml'
 require 'open-uri'
 require 'pretty_strings'
 
-Encoding.default_internal = Encoding::UTF_8
-Encoding.default_external = Encoding::UTF_8
-
 module TmxImporter
   class Tmx
     attr_reader :file_path, :encoding
@@ -18,6 +15,12 @@ module TmxImporter
         seg: { lang: "", counter: 0, vals: [], role: "" },
         language_pairs: []
       }
+      @src_regex = Regexp.new('(?<=srclang=\S)\S+(?=")|(?=\')'.encode(@encoding))
+      @src_string = 'srclang='.encode(@encoding).freeze
+      @tu_regex = Regexp.new('<\/tu>'.encode(@encoding))
+      @seg_regex = Regexp.new('<\/seg>'.encode(@encoding))
+      @lang_regex = Regexp.new('(?<=[^cn]lang=\S)\S+(?=")|(?=\')'.encode(@encoding))
+      @lang_string = 'lang'.encode(@encoding).freeze
       raise "Encoding type not supported. Please choose an encoding of UTF-8, UTF-16LE, or UTF-16BE" unless @encoding.eql?('UTF-8') || @encoding.eql?('UTF-16LE') || @encoding.eql?('UTF-16BE')
     end
 
@@ -43,11 +46,12 @@ module TmxImporter
     end
 
     def analyze_line(line)
-      @doc[:source_language] = line.scan(/(?<=srclang=\S)\S+(?=")|(?=')/)[0] if line.include?('srclang=')
-      @doc[:tu][:counter] += line.scan(/<\/tu>/).count
-      @doc[:seg][:counter] += line.scan(/<\/seg>/).count
-      if line.include?('lang')
-        @doc[:seg][:lang] = line.scan(/(?<=[^cn]lang=\S)\S+(?=")|(?=')/)[0]
+      @doc[:source_language] = line.scan(@src_regex)[0].encode('UTF-8') if line.include?(@src_string)
+      @doc[:tu][:counter] += line.scan(@tu_regex).count
+      @doc[:seg][:counter] += line.scan(@seg_regex).count
+      if line.include?(@lang_string)
+        @doc[:seg][:lang] = line.scan(@lang_regex)[0]
+        @doc[:seg][:lang] = @doc[:seg][:lang].encode('UTF-8') unless @doc[:seg][:lang].nil?
         write_language_pair
       end
     end
@@ -105,7 +109,7 @@ module TmxImporter
       if @doc[:seg][:lang] != @doc[:source_language] &&
          @doc[:seg][:lang].split('-')[0].downcase != @doc[:source_language].split('-')[0].downcase &&
          @doc[:source_language] != '*all*'
-        @doc[:language_pairs] << [@doc[:source_language], @doc[:seg][:lang]]
+        @doc[:language_pairs] << [@doc[:source_language].force_encoding("UTF-8"), @doc[:seg][:lang].force_encoding("UTF-8")]
         @doc[:seg][:role] = 'source'
       elsif @doc[:source_language] == '*all*'
         @doc[:source_language] = @doc[:seg][:lang]
@@ -117,7 +121,7 @@ module TmxImporter
 
     def write_tu(reader)
       @doc[:tu][:lang] = reader.get_attribute("srclang")
-      @doc[:tu][:creation_date] = reader.get_attribute("creationdate").nil? ? DateTime.now.to_s : DateTime.parse(reader.get_attribute("creationdate")).to_s
+      @doc[:tu][:creation_date] = reader.get_attribute("creationdate").nil? ? DateTime.now.to_s : DateTime.parse(reader.get_attribute("creationdate").force_encoding('UTF-8')).to_s
       @doc[:tu][:vals] << [@doc[:tu][:id], @doc[:tu][:creation_date]]
     end
 

data/spec/tmx_importer_spec.rb

@@ -67,6 +67,12 @@ describe TmxImporter do
       tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
       expect(tmx.stats).to eq({:tu_count=>4, :seg_count=>8, :language_pairs=>[["de-DE", "en-US"]]})
     end
+
+    it 'imports a TMX file with UTF-16 LE BOM encoding' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/strange_encoding.tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-16LE')
+      expect(tmx.stats).to eq({:tu_count=>1, :seg_count=>2, :language_pairs=>[["tr", "en"]]})
+    end
   end
 
   describe '#import' do
@@ -117,5 +123,11 @@ describe TmxImporter do
       tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
       expect(tmx.import[1][2][1]).to eq("target")
     end
+
+    it 'imports a TMX file with UTF-16 LE BOM encoding' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/strange_encoding.tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-16LE')
+      expect(tmx.import[1][1][3]).to eq("en")
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tmx_importer
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Kevin S. Dias
@@ -103,6 +103,7 @@ files:
 - spec/test_sample_files/multiple_language_pairs.tmx
 - spec/test_sample_files/out_of_order_segments.tmx
 - spec/test_sample_files/srclang_all.tmx
+- spec/test_sample_files/strange_encoding.tmx
 - spec/test_sample_files/test_tm(utf-16LE BOM).tmx
 - spec/test_sample_files/test_tm(utf-16LE).tmx
 - spec/test_sample_files/test_tm(utf-8).tmx
@@ -140,6 +141,7 @@ test_files:
 - spec/test_sample_files/multiple_language_pairs.tmx
 - spec/test_sample_files/out_of_order_segments.tmx
 - spec/test_sample_files/srclang_all.tmx
+- spec/test_sample_files/strange_encoding.tmx
 - spec/test_sample_files/test_tm(utf-16LE BOM).tmx
 - spec/test_sample_files/test_tm(utf-16LE).tmx
 - spec/test_sample_files/test_tm(utf-8).tmx