tmx_importer 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77a255ff1cd88f6d3edbc6748e2680ec86486abf
-  data.tar.gz: 01001fd8f8c612fffcc83e110b77d9738299e046
+  metadata.gz: 07447d45355fb271c832db8228367a0d14d04c5f
+  data.tar.gz: ab468fdf9295379cd036b351648f37eb3477468f
 SHA512:
-  metadata.gz: da87359b112880242cca91b25005b828017d7563b2915217759576da7a8b222e29cfce2b7df55fd3196075c25ab1168cb3f5de4c4cb73dc25958b1075a9c7a3a
-  data.tar.gz: ae1aade7c0e33859dd4f6dcceebb5171d345ef5f7dececcd3e5e57f4657f86162ceb752ce0ebc0511b4d38ebccee0e49810c1b745727ac75b1463ea9aa947c89
+  metadata.gz: 508a95ae93f7a43af68362b97ab429d5f746486ad2bc2216174d35490b1c9952e8059e5eb111074c05ba40055d1e255d34d23439caae5ebfcebfa6dcceb0116e
+  data.tar.gz: 5aa98d1fb3a4ed65554f149c9d8be21fc2a3fe2b9cc02034bb63718ec1a7ec68049da278c9793d036cf450aaf7099ce1799eccc01e554e724d69e52614d11daa

data/lib/tmx_importer/version.rb

@@ -1,3 +1,3 @@
 module TmxImporter
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/tmx_importer.rb

@@ -2,8 +2,7 @@ require 'tmx_importer/version'
 require 'xml'
 require 'open-uri'
 require 'pretty_strings'
-
-Encoding.default_external = Encoding::UTF_8
+require 'charlock_holmes'
 
 module TmxImporter
   class Tmx
@@ -17,20 +16,15 @@ module TmxImporter
         seg: { lang: "", counter: 0, vals: [], role: "" },
         language_pairs: []
       }
-      @src_regex = Regexp.new('(?<=srclang=\S)\S+(?=")|(?=\')'.encode(@encoding))
-      @src_string = 'srclang='.encode(@encoding).freeze
-      @tu_regex = Regexp.new('<\/tu>'.encode(@encoding))
-      @seg_regex = Regexp.new('<\/seg>'.encode(@encoding))
-      @lang_regex = Regexp.new('(?<=[^cn]lang=\S)\S+(?=")|(?=\')'.encode(@encoding))
-      @lang_string = 'lang'.encode(@encoding).freeze
+      @text = CharlockHolmes::Converter.convert(File.read(open(@file_path)), encoding, 'UTF-8') if !@encoding.eql?('UTF-8')
       raise "Encoding type not supported. Please choose an encoding of UTF-8, UTF-16LE, or UTF-16BE" unless @encoding.eql?('UTF-8') || @encoding.eql?('UTF-16LE') || @encoding.eql?('UTF-16BE')
     end
 
     def stats
-      File.open(@file_path, "rb:#{encoding}") do |file|
-        file.each do |line|
-          analyze_line(line)
-        end
+      if encoding.eql?('UTF-8')
+        analyze_stats_utf_8
+      else
+        analyze_stats_utf_16
       end
       {tu_count: @doc[:tu][:counter], seg_count: @doc[:seg][:counter], language_pairs: @doc[:language_pairs].uniq}
     end
@@ -43,29 +37,35 @@ module TmxImporter
 
     private
 
-    def read_file
-      XML::Reader.io(open(file_path), options: XML::Parser::Options::NOERROR, encoding: set_encoding)
+    def analyze_stats_utf_8
+      File.readlines(@file_path).each do |line|
+        analyze_line(line)
+      end
     end
 
-    def analyze_line(line)
-      @doc[:source_language] = line.scan(@src_regex)[0].encode('UTF-8') if line.include?(@src_string)
-      @doc[:tu][:counter] += line.scan(@tu_regex).count
-      @doc[:seg][:counter] += line.scan(@seg_regex).count
-      if line.include?(@lang_string)
-        @doc[:seg][:lang] = line.scan(@lang_regex)[0]
-        @doc[:seg][:lang] = @doc[:seg][:lang].encode('UTF-8') unless @doc[:seg][:lang].nil?
-        write_language_pair
+    def analyze_stats_utf_16
+      @text.each_line do |line|
+        analyze_line(line)
       end
     end
 
-    def set_encoding
-      case encoding
-      when 'UTF-8'
-        xml_encoding = XML::Encoding::UTF_8
-      when 'UTF-16LE'
-        xml_encoding = XML::Encoding::UTF_16LE
-      when 'UTF-16BE'
-        xml_encoding = XML::Encoding::UTF_16BE
+    def read_file
+      if encoding.eql?('UTF-8')
+        XML::Reader.io(open(file_path), options: XML::Parser::Options::NOERROR, encoding: XML::Encoding::UTF_8)
+      else
+        reader = @text.gsub!(/(?<=encoding=").*(?=")/, 'utf-8').gsub(/&#x[0-1]?[0-9a-fA-F];/, ' ').gsub(/[\0-\x1f\x7f\u2028]/, ' ')
+        XML::Reader.string(reader, options: XML::Parser::Options::NOERROR, encoding: XML::Encoding::UTF_8)
+      end
+    end
+
+    def analyze_line(line)
+      @doc[:source_language] = line.scan(/(?<=srclang=\S)\S+(?=")|(?=')/)[0] if line.include?('srclang=')
+      @doc[:tu][:counter] += line.scan(/<\/tu>/).count
+      @doc[:seg][:counter] += line.scan(/<\/seg>/).count
+      if line.include?('lang')
+        @doc[:seg][:lang] = line.scan(/(?<=[^cn]lang=\S)\S+(?=")|(?=')/)[0]
+        @doc[:seg][:lang] = @doc[:seg][:lang] unless @doc[:seg][:lang].nil?
+        write_language_pair
       end
     end
 
@@ -92,13 +92,13 @@ module TmxImporter
     def eval_state_initial(tag_stack, reader)
       case tag_stack.last.bytes.to_a
       when [104, 101, 97, 100, 101, 114]
-        @doc[:source_language] = reader.get_attribute("srclang").force_encoding("UTF-8") if @doc[:source_language].empty? && reader.has_attributes? && reader.get_attribute("srclang")
+        @doc[:source_language] = reader.get_attribute("srclang") if @doc[:source_language].empty? && reader.has_attributes? && reader.get_attribute("srclang")
       when [116, 117]
         write_tu(reader)
         @doc[:tu][:counter] += 1
       when [116, 117, 118]
         seg_lang = reader.get_attribute("lang") || reader.get_attribute("xml:lang")
-        @doc[:seg][:lang] = seg_lang.force_encoding("UTF-8") unless seg_lang.empty?
+        @doc[:seg][:lang] = seg_lang unless seg_lang.empty?
       when [115, 101, 103]
         write_seg(reader)
         write_language_pair
@@ -111,10 +111,10 @@ module TmxImporter
       if @doc[:seg][:lang] != @doc[:source_language] &&
          @doc[:seg][:lang].split('-')[0].downcase != @doc[:source_language].split('-')[0].downcase &&
          @doc[:source_language] != '*all*'
-        @doc[:language_pairs] << [@doc[:source_language].force_encoding("UTF-8"), @doc[:seg][:lang].force_encoding("UTF-8")]
+        @doc[:language_pairs] << [@doc[:source_language], @doc[:seg][:lang]]
         @doc[:seg][:role] = 'source'
       elsif @doc[:source_language] == '*all*'
-        @doc[:source_language] = @doc[:seg][:lang].force_encoding("UTF-8")
+        @doc[:source_language] = @doc[:seg][:lang]
         @doc[:seg][:role] = 'source'
       else
         @doc[:seg][:role] = 'target'
@@ -123,13 +123,13 @@ module TmxImporter
 
     def write_tu(reader)
       @doc[:tu][:lang] = reader.get_attribute("srclang")
-      @doc[:tu][:creation_date] = reader.get_attribute("creationdate").nil? ? DateTime.now.to_s : DateTime.parse(reader.get_attribute("creationdate").force_encoding('UTF-8')).to_s
+      @doc[:tu][:creation_date] = reader.get_attribute("creationdate").nil? ? DateTime.now.to_s : DateTime.parse(reader.get_attribute("creationdate")).to_s
       @doc[:tu][:vals] << [@doc[:tu][:id], @doc[:tu][:creation_date]]
     end
 
     def write_seg(reader)
       return if reader.read_string.nil?
-      text = PrettyStrings::Cleaner.new(reader.read_string.force_encoding('UTF-8')).pretty.gsub("\\","&#92;").gsub("'",%q(\\\'))
+      text = PrettyStrings::Cleaner.new(reader.read_string).pretty.gsub("\\","&#92;").gsub("'",%q(\\\'))
       word_count = text.gsub("\s+", ' ').split(' ').length
       @doc[:seg][:vals] << [@doc[:tu][:id], @doc[:seg][:role], word_count, @doc[:seg][:lang], text, @doc[:tu][:creation_date]]
     end

data/tmx_importer.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec"
   spec.add_runtime_dependency "libxml-ruby"
   spec.add_runtime_dependency "pretty_strings", "~> 0.5.0"
+  spec.add_runtime_dependency "charlock_holmes_bundle_icu", '~> 0.6.9.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tmx_importer
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-11 00:00:00.000000000 Z
+date: 2016-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.0
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes_bundle_icu
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.9.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.9.2
 description: Import the content of a TMX translation memory file to your database
 email:
 - diasks2@gmail.com