tmx_importer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 99a80fae6e289623de257e8383f5ba8e6d90eda5
+  data.tar.gz: 150248c983d19427e7cf3ccbfc98c4a63227436f
+SHA512:
+  metadata.gz: 9b869fc95d11ba967d43ef4f54914d557d10bbaa57c44831ed86e4fe36787159262a27b6c2dbcdec3902dc4936a1e7e72a6327f10d30a2576a45487dc5e7432d
+  data.tar.gz: 0c11e503f9447569d90aaf49199691fd7bb54bd9d24b0a4371d8242259b9353183da08dbe6c751fa10d36c9488b13aa8f2b1c0886c0a06f1cf62f703eef1333d

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.1.2
+  - 2.2.4

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tmx_importer.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Kevin S. Dias
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,70 @@
+# TMX Importer
+
+[![Gem Version](https://badge.fury.io/rb/tmx_importer.svg)](https://badge.fury.io/rb/tmx_importer) [![Build Status](https://travis-ci.org/diasks2/tmx_importer.png)](https://travis-ci.org/diasks2/tmx_importer) [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/diasks2/tmx_importer/blob/master/LICENSE.txt)
+
+This gem handles the importing and parsing of [.tmx translation memory files](http://www.ttt.org/oscarstandards/tmx/tmx14-20020710.htm). TMX files are xml files.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+**Ruby**  
+```
+gem install tmx_importer
+```
+
+**Ruby on Rails**  
+Add this line to your application’s Gemfile:  
+```ruby 
+gem 'tmx_importer'
+```
+
+## Usage
+
+```ruby
+# Get the high level stats of a TMX file
+file_path = File.expand_path('../tmx_importer/spec/test_sample_files/test_tm(utf-8).tmx')
+TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8').stats
+# => {:tu_count=>4, :seg_count=>8, :language_pairs=>[["de-DE", "en-US"]]}
+
+# Extract the segments of a TMX file
+# Result: [translation_units, segments]
+# translation_units = [tu_id, creation_date]
+# segments = [tu_id, segment_role, word_count, language, segment_text]
+
+file_path = File.expand_path('../tmx_importer/spec/test_sample_files/test_tm(utf-8).tmx')
+TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8').import
+# => [[["5533-1457670156-1", "2016-03-11T13:22:36+09:00"], ["6836-1457670156-3", "2016-03-11T13:22:36+09:00"], ["3285-1457670156-5", "2016-03-11T13:22:36+09:00"], ["6706-1457670156-7", "2016-03-11T13:22:36+09:00"]], [["5533-1457670156-1", "", 1, "de-DE", "überprüfen"], ["5533-1457670156-1", "target", 1, "en-US", "check"], ["6836-1457670156-3", "source", 1, "de-DE", "Rückenlehneneinstellung"], ["6836-1457670156-3", "target", 2, "en-US", "Backrest adjustment"], ["3285-1457670156-5", "source", 1, "de-DE", "Bezüglich"], ["3285-1457670156-5", "target", 3, "en-US", "In terms of"], ["6706-1457670156-7", "source", 20, "de-DE", "Der Staatsschutz prüft, ob es einen Zusammenhang mit einem Anschlag auf eine geplante Flüchtlingsunterkunft in der Nachbarschaft Ende August gibt."], ["6706-1457670156-7", "target", 23, "en-US", "The state protection checks whether there is a connection with an attack on a planned refugee camp in the neighborhood of late August."]]]
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/diasks2/tmx_importer/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## License
+
+The MIT License (MIT)
+
+Copyright (c) 2016 Kevin S. Dias
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,5 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+task default: :spec

data/lib/tmx_importer/version.rb

@@ -0,0 +1,3 @@
+module TmxImporter
+  VERSION = "0.1.0"
+end

data/lib/tmx_importer.rb

@@ -0,0 +1,135 @@
+require 'tmx_importer/version'
+require 'xml'
+require 'open-uri'
+require 'pretty_strings'
+
+Encoding.default_internal = Encoding::UTF_8
+Encoding.default_external = Encoding::UTF_8
+
+module TmxImporter
+  class Tmx
+    attr_reader :file_path, :encoding
+    def initialize(file_path:, encoding:)
+      @file_path = file_path
+      @encoding = encoding.upcase
+      @doc = {
+        source_language: "",
+        tu: { id: "", counter: 0, vals: [], lang: "" },
+        seg: { lang: "", counter: 0, vals: [], role: "" },
+        language_pairs: []
+      }
+      raise "Encoding type not supported. Please choose an encoding of UTF-8, UTF-16LE, or UTF-16BE" unless @encoding.eql?('UTF-8') || @encoding.eql?('UTF-16LE') || @encoding.eql?('UTF-16BE')
+    end
+
+    def stats
+      File.open(@file_path, "rb:#{encoding}") do |file|
+        file.each do |line|
+          analyze_line(line)
+        end
+      end
+      {tu_count: @doc[:tu][:counter], seg_count: @doc[:seg][:counter], language_pairs: @doc[:language_pairs].uniq}
+    end
+
+    def import
+      reader = read_file
+      parse_file(reader)
+      [@doc[:tu][:vals], @doc[:seg][:vals]]
+    end
+
+    private
+
+    def read_file
+      XML::Reader.io(open(file_path), options: XML::Parser::Options::NOERROR, encoding: set_encoding)
+    end
+
+    def analyze_line(line)
+      @doc[:source_language] = line.scan(/(?<=srclang=\S)\S+(?=")|(?=')/)[0] if line.include?('srclang=')
+      @doc[:tu][:counter] += line.scan(/<\/tu>/).count
+      @doc[:seg][:counter] += line.scan(/<\/seg>/).count
+      if line.include?('lang')
+        @doc[:seg][:lang] = line.scan(/(?<=[^cn]lang=\S)\S+(?=")|(?=')/)[0]
+        write_language_pair
+      end
+    end
+
+    def set_encoding
+      case encoding
+      when 'UTF-8'
+        xml_encoding = XML::Encoding::UTF_8
+      when 'UTF-16LE'
+        xml_encoding = XML::Encoding::UTF_16LE
+      when 'UTF-16BE'
+        xml_encoding = XML::Encoding::UTF_16BE
+      end
+    end
+
+    def parse_file(reader)
+      tag_stack = []
+      generate_unique_id
+      while reader.read do
+        tag_stack.delete_if { |d| d.bytes.to_a == [101, 112, 116] ||
+                                  d.bytes.to_a == [98, 112, 116] ||
+                                  d.bytes.to_a == [112, 114, 111, 112] ||
+                                  d.bytes.to_a == [112, 104] }
+        if !tag_stack.include?(reader.name)
+          tag_stack.push(reader.name)
+          eval_state_initial(tag_stack, reader)
+        elsif tag_stack.last == reader.name
+          d = tag_stack.dup.pop
+          tag_stack.pop if d.bytes.to_a == [35, 116, 101, 120, 116]
+          generate_unique_id if tag_stack.length > 3 && tag_stack.pop.bytes.to_a == [116, 117]
+        end
+      end
+      reader.close
+    end
+
+    def eval_state_initial(tag_stack, reader)
+      case tag_stack.last.bytes.to_a
+      when [104, 101, 97, 100, 101, 114]
+        @doc[:source_language] = reader.get_attribute("srclang").force_encoding("UTF-8") if @doc[:source_language].empty? && reader.has_attributes? && reader.get_attribute("srclang")
+      when [116, 117]
+        write_tu(reader)
+        @doc[:tu][:counter] += 1
+      when [116, 117, 118]
+        seg_lang = reader.get_attribute("lang") || reader.get_attribute("xml:lang")
+        @doc[:seg][:lang] = seg_lang.force_encoding("UTF-8") unless seg_lang.empty?
+      when [115, 101, 103]
+        write_seg(reader)
+        write_language_pair
+        @doc[:seg][:counter] += 1
+      end
+    end
+
+    def write_language_pair
+      return if @doc[:seg][:lang].nil? || @doc[:seg][:lang].empty? || @doc[:source_language].nil? || @doc[:source_language].empty?
+      if @doc[:seg][:lang] != @doc[:source_language] &&
+         @doc[:seg][:lang].split('-')[0].downcase != @doc[:source_language].split('-')[0].downcase &&
+         @doc[:source_language] != '*all*'
+        @doc[:language_pairs] << [@doc[:source_language], @doc[:seg][:lang]]
+        @doc[:seg][:role] = 'source'
+      elsif @doc[:source_language] == '*all*'
+        @doc[:source_language] = @doc[:seg][:lang]
+        @doc[:seg][:role] = 'source'
+      else
+        @doc[:seg][:role] = 'target'
+      end
+    end
+
+    def write_tu(reader)
+      @doc[:tu][:lang] = reader.get_attribute("srclang")
+      created_date = reader.get_attribute("creationdate").nil? ? DateTime.now.to_s : DateTime.parse(reader.get_attribute("creationdate")).to_s
+      @doc[:tu][:vals] << [@doc[:tu][:id], created_date]
+    end
+
+    def write_seg(reader)
+      return if reader.read_string.empty?
+      text = PrettyStrings::Cleaner.new(reader.read_string.force_encoding('UTF-8')).pretty.gsub("\\","&#92;").gsub("'",%q(\\\'))
+      word_count = text.gsub("\s+", ' ').split(' ').length
+      @doc[:seg][:vals] << [@doc[:tu][:id], @doc[:seg][:role], word_count, @doc[:seg][:lang], text]
+    end
+
+    def generate_unique_id
+      @doc[:tu][:id] = [(1..4).map{rand(10)}.join(''), Time.now.to_i, @doc[:tu][:counter] += 1 ].join("-")
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'tmx_importer'

data/spec/test_sample_files/bad_encoding.tmx

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<tmx version="1.4">
+  <header creationtool="AwesomeTool" creationtoolversion="2.0" datatype="plaintext" segtype="sentence" o-tmf="AwesomeTool TMX" adminlang="EN-US" srclang="de-DE"></header>
+  <body>
+    <tu tuid="1">
+      <tuv xml:lang="de-DE">
+        <seg>�berpr�fen</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>check</seg>
+      </tuv>
+    </tu>
+    <tu tuid="2">
+      <tuv xml:lang="de-DE">
+        <seg>R�ckenlehneneinstellung</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>Backrest adjustment</seg>
+      </tuv>
+    </tu>
+    <tu tuid="3">
+      <tuv xml:lang="de-DE">
+        <seg>Bez�glich</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>In terms of</seg>
+      </tuv>
+    </tu>
+    <tu tuid="4">
+      <tuv xml:lang="de-DE">
+        <seg>Der Staatsschutz pr�ft, ob es einen Zusammenhang mit einem Anschlag auf eine geplante Fl�chtlingsunterkunft in der Nachbarschaft Ende August gibt.</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>The state protection checks whether there is a connection with an attack on a planned refugee camp in the neighborhood of late August.</seg>
+      </tuv>
+    </tu>
+  </body>
+</tmx>

data/spec/test_sample_files/bad_markup(utf-8).tmx

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE tmx SYSTEM "tmx11.dtd">
+<tmx version="1.1">
+  <header
+  	 creationtool="AwesomeTool"
+  	 creationtoolversion="Edition 8 Build 863"
+  	 segtype="sentence"
+  	 o-tmf="TW4Win 2.0 Format"
+  	 adminlang="EN-US"
+  	 srclang="EN-US"
+  	 datatype="rtf"
+  	 creationdate="20110527T214247Z"
+  	 creationid="A"
+  >
+  <prop type="RTFFontTable">
+  {\fonttbl
+  {\f1 \fmodern\fprq1 \fcharset0 Courier New;}
+  {\f2 \fswiss\fprq2 \fcharset0 Arial;}
+  {\f3 \fcharset238 Arial CE;}
+  {\f4 \fmodern\fprq1 \fcharset238 Courier New CE;}
+  {\f5 \fcharset0 Arial Unicode MS;}
+  {\f6 \fcharset238 Arial Unicode MS CE;}
+  {\f7 \fcharset238 Tahoma CE;}
+  {\f8 \fcharset0 Tahoma;}}</prop>
+  <prop type="RTFStyleSheet">
+  {\stylesheet
+  {\St \s0 {\StN Normal}}
+  {\St \cs1 {\StB \v\f1\fs24\sub\cf12 }{\StN tw4winMark}}
+  {\St \cs2 {\StB \cf4\fs40\f1 }{\StN tw4winError}}
+  {\St \cs3 {\StB \f1\cf11\lang1024 }{\StN tw4winPopup}}
+  {\St \cs4 {\StB \f1\cf10\lang1024 }{\StN tw4winJump}}
+  {\St \cs5 {\StB \f1\cf15\lang1024 }{\StN tw4winExternal}}
+  {\St \cs6 {\StB \f1\cf6\lang1024 }{\StN tw4winInternal}}
+  {\St \cs7 {\StB \cf2 }{\StN tw4winTerm}}
+  {\St \cs8 {\StB \f1\cf13\lang1024 }{\StN DO_NOT_TRANSLATE}}}</prop>
+  </header>
+
+  <body>
+    <tu tuid=""1"">
+      <tuv xml:lang="de-DE">
+        <seg>überprüfen</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>check</seg>
+      </tuv>
+    </tu>
+    <tu tuid="2">
+      <tuv xml:lang="de-DE">
+        <seg>Rückenlehneneinstellung</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>Backrest adjustment</seg>
+      </tuv>
+    </tu>
+    <tu tuid="3">
+      <tuv xml:lang="de-DE">
+        <seg>Bezüglich</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>In terms of</seg>
+      </tuv>
+    </tu>
+    <tu tuid="4">
+      <tuv xml:lang="de-DE">
+        <seg>Der Staatsschutz prüft, ob es einen Zusammenhang mit einem Anschlag auf eine geplante Flüchtlingsunterkunft in der Nachbarschaft Ende August gibt.</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>The state protection checks whether there is a connection with an attack on a planned refugee camp in the neighborhood of late August.</seg>
+      </tuv>
+    </tu>
+  </body>
+</tmx>

data/spec/test_sample_files/multiple_language_pairs.tmx

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+<tmx version="1.4">
+  <header creationtool="AwesomeTool" creationtoolversion="2.0" datatype="plaintext" segtype="sentence" o-tmf="AwesomeTool TMX" adminlang="EN-US" srclang="de-DE"></header>
+  <body>
+    <tu tuid="1">
+      <tuv xml:lang="de-DE">
+        <seg>überprüfen</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>check</seg>
+      </tuv>
+      <tuv xml:lang="it">
+        <seg>controllare</seg>
+      </tuv>
+      <tuv xml:lang="fr">
+        <seg>vérifier</seg>
+      </tuv>
+    </tu>
+    <tu tuid="2">
+      <tuv xml:lang="de-DE">
+        <seg>Rückenlehneneinstellung</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>Backrest adjustment</seg>
+      </tuv>
+    </tu>
+    <tu tuid="3">
+      <tuv xml:lang="de-DE">
+        <seg>Bezüglich</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>In terms of</seg>
+      </tuv>
+    </tu>
+    <tu tuid="4">
+      <tuv xml:lang="de-DE">
+        <seg>Der Staatsschutz prüft, ob es einen Zusammenhang mit einem Anschlag auf eine geplante Flüchtlingsunterkunft in der Nachbarschaft Ende August gibt.</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>The state protection checks whether there is a connection with an attack on a planned refugee camp in the neighborhood of late August.</seg>
+      </tuv>
+    </tu>
+  </body>
+</tmx>

data/spec/test_sample_files/out_of_order_segments.tmx

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<tmx version="1.4">
+  <header creationtool="AwesomeTool" creationtoolversion="2.0" datatype="plaintext" segtype="sentence" o-tmf="AwesomeTool TMX" adminlang="EN-US" srclang="de-DE"></header>
+  <body>
+    <tu tuid="1">
+      <tuv xml:lang="en-US">
+        <seg>check</seg>
+      </tuv>
+      <tuv xml:lang="de-DE">
+        <seg>überprüfen</seg>
+      </tuv>
+    </tu>
+    <tu tuid="2">
+      <tuv xml:lang="en-US">
+        <seg>Backrest adjustment</seg>
+      </tuv>
+      <tuv xml:lang="de-DE">
+        <seg>Rückenlehneneinstellung</seg>
+      </tuv>
+    </tu>
+    <tu tuid="3">
+      <tuv xml:lang="en-US">
+        <seg>In terms of</seg>
+      </tuv>
+      <tuv xml:lang="de-DE">
+        <seg>Bezüglich</seg>
+      </tuv>
+    </tu>
+    <tu tuid="4">
+      <tuv xml:lang="en-US">
+        <seg>The state protection checks whether there is a connection with an attack on a planned refugee camp in the neighborhood of late August.</seg>
+      </tuv>
+      <tuv xml:lang="de-DE">
+        <seg>Der Staatsschutz prüft, ob es einen Zusammenhang mit einem Anschlag auf eine geplante Flüchtlingsunterkunft in der Nachbarschaft Ende August gibt.</seg>
+      </tuv>
+    </tu>
+  </body>
+</tmx>

data/spec/test_sample_files/srclang_all.tmx

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+<tmx version="1.4">
+  <header creationtool="AwesomeTool" creationtoolversion="2.0" datatype="plaintext" segtype="sentence" o-tmf="AwesomeTool TMX" adminlang="EN-US" srclang="*all*"></header>
+  <body>
+    <tu tuid="1">
+      <tuv xml:lang="de-DE">
+        <seg>überprüfen</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>check</seg>
+      </tuv>
+      <tuv xml:lang="it">
+        <seg>controllare</seg>
+      </tuv>
+      <tuv xml:lang="fr">
+        <seg>vérifier</seg>
+      </tuv>
+    </tu>
+    <tu tuid="2">
+      <tuv xml:lang="de-DE">
+        <seg>Rückenlehneneinstellung</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>Backrest adjustment</seg>
+      </tuv>
+    </tu>
+    <tu tuid="3">
+      <tuv xml:lang="de-DE">
+        <seg>Bezüglich</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>In terms of</seg>
+      </tuv>
+    </tu>
+    <tu tuid="4">
+      <tuv xml:lang="de-DE">
+        <seg>Der Staatsschutz prüft, ob es einen Zusammenhang mit einem Anschlag auf eine geplante Flüchtlingsunterkunft in der Nachbarschaft Ende August gibt.</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>The state protection checks whether there is a connection with an attack on a planned refugee camp in the neighborhood of late August.</seg>
+      </tuv>
+    </tu>
+  </body>
+</tmx>

data/spec/test_sample_files/test_tm(utf-8).tmx

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<tmx version="1.4">
+  <header creationtool="AwesomeTool" creationtoolversion="2.0" datatype="plaintext" segtype="sentence" o-tmf="AwesomeTool TMX" adminlang="EN-US" srclang="de-DE"></header>
+  <body>
+    <tu tuid="1">
+      <tuv xml:lang="de-DE">
+        <seg>überprüfen</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>check</seg>
+      </tuv>
+    </tu>
+    <tu tuid="2">
+      <tuv xml:lang="de-DE">
+        <seg>Rückenlehneneinstellung</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>Backrest adjustment</seg>
+      </tuv>
+    </tu>
+    <tu tuid="3">
+      <tuv xml:lang="de-DE">
+        <seg>Bezüglich</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>In terms of</seg>
+      </tuv>
+    </tu>
+    <tu tuid="4">
+      <tuv xml:lang="de-DE">
+        <seg>Der Staatsschutz prüft, ob es einen Zusammenhang mit einem Anschlag auf eine geplante Flüchtlingsunterkunft in der Nachbarschaft Ende August gibt.</seg>
+      </tuv>
+      <tuv xml:lang="en-US">
+        <seg>The state protection checks whether there is a connection with an attack on a planned refugee camp in the neighborhood of late August.</seg>
+      </tuv>
+    </tu>
+  </body>
+</tmx>

data/spec/test_sample_files/test_tm_2(utf-8).tmx

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<tmx version="1.4">
+  <header creationtool="AwesomeTool" creationtoolversion="2.0" datatype="plaintext" segtype="sentence" o-tmf="AwesomeTool TMX" adminlang="EN" srclang="de"></header>
+  <body>
+    <tu tuid="1">
+      <tuv xml:lang="de">
+        <seg>überprüfen</seg>
+      </tuv>
+      <tuv xml:lang="en">
+        <seg>check</seg>
+      </tuv>
+    </tu>
+    <tu tuid="2">
+      <tuv xml:lang="de">
+        <seg>Rückenlehneneinstellung</seg>
+      </tuv>
+      <tuv xml:lang="en">
+        <seg>Backrest adjustment</seg>
+      </tuv>
+    </tu>
+    <tu tuid="3">
+      <tuv xml:lang="de-DE">
+        <seg>Bezüglich</seg>
+      </tuv>
+      <tuv xml:lang="en">
+        <seg>In terms of</seg>
+      </tuv>
+    </tu>
+    <tu tuid="4">
+      <tuv xml:lang="de">
+        <seg>Der Staatsschutz prüft, ob es einen Zusammenhang mit einem Anschlag auf eine geplante Flüchtlingsunterkunft in der Nachbarschaft Ende August gibt.</seg>
+      </tuv>
+      <tuv xml:lang="en">
+        <seg>The state protection checks whether there is a connection with an attack on a planned refugee camp in the neighborhood of late August.</seg>
+      </tuv>
+    </tu>
+  </body>
+</tmx>

data/spec/tmx_importer_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+
+describe TmxImporter do
+  it 'has a version number' do
+    expect(TmxImporter::VERSION).not_to be nil
+  end
+
+  it 'raises an error if the encoding is not supported' do
+    file_path = File.expand_path('../tmx_importer/spec/test_sample_files/test_tm(utf-8).tmx')
+    -> { expect(TmxImporter::Tmx.new(file_path: file_path, encoding: 'ISO-8859-9').stats).to raise_error }
+  end
+
+  it 'raises an error if the wrong encoding is specified in the file' do
+    file_path = File.expand_path('../tmx_importer/spec/test_sample_files/bad_encoding.tmx')
+    -> { expect(TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8').stats).to raise_error }
+  end
+
+  it 'raises an error if the file contains bad markup' do
+    file_path = File.expand_path('../tmx_importer/spec/test_sample_files/bad_markup(utf-8).tmx')
+    -> { expect(TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8').stats).to raise_error }
+  end
+
+  it 'raises an error if the file contains bad markup 2' do
+    file_path = File.expand_path('../tmx_importer/spec/test_sample_files/bad_markup(utf-16).tmx')
+    -> { expect(TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-16le').stats).to raise_error }
+  end
+
+  describe '#stats' do
+    it 'reports the stats of a UTF-8 TMX file' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/test_tm(utf-8).tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
+      expect(tmx.stats).to eq({:tu_count=>4, :seg_count=>8, :language_pairs=>[["de-DE", "en-US"]]})
+    end
+
+    it 'reports the stats of a UTF-8 TMX file 2' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/test_tm_2(utf-8).tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
+      expect(tmx.stats).to eq({:tu_count=>4, :seg_count=>8, :language_pairs=>[["de", "en"]]})
+    end
+
+    it 'reports the stats of a UTF-16LE TMX file' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/test_tm(utf-16LE).tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-16le')
+      expect(tmx.stats).to eq({:tu_count=>4, :seg_count=>8, :language_pairs=>[["de-DE", "en-US"]]})
+    end
+
+    it 'reports the stats of a UTF-16LE BOM TMX file' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/test_tm(utf-16LE BOM).tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-16le')
+      expect(tmx.stats).to eq({:tu_count=>4, :seg_count=>8, :language_pairs=>[["de-DE", "en-US"]]})
+    end
+
+    it 'reports the stats of a multiple language pair TMX file' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/multiple_language_pairs.tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
+      expect(tmx.stats).to eq({:tu_count=>4, :seg_count=>10, :language_pairs=>[["de-DE", "en-US"], ["de-DE", "it"], ["de-DE", "fr"]]})
+    end
+
+    it 'reports the stats of a srclang equals *all* TMX file' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/srclang_all.tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
+      expect(tmx.stats).to eq({:tu_count=>4, :seg_count=>10, :language_pairs=>[["de-DE", "en-US"], ["de-DE", "it"], ["de-DE", "fr"]]})
+    end
+
+    it 'reports the stats of a TMX file with out of order segments' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/out_of_order_segments.tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
+      expect(tmx.stats).to eq({:tu_count=>4, :seg_count=>8, :language_pairs=>[["de-DE", "en-US"]]})
+    end
+  end
+
+  describe '#import' do
+    it 'imports a UTF-8 TMX file' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/test_tm(utf-8).tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8').import
+      expect(tmx[1][2][3]).to eq("de-DE")
+    end
+
+    it 'imports a UTF-8 TMX file 2' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/test_tm_2(utf-8).tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
+      expect(tmx.import[1][2][4]).to eq("Rückenlehneneinstellung")
+    end
+
+    it 'imports a UTF-16LE TMX file' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/test_tm(utf-16LE).tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-16le')
+      expect(tmx.import[1][3][4]).to eq("Backrest adjustment")
+    end
+
+    it 'imports a UTF-16LE BOM TMX file' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/test_tm(utf-16LE BOM).tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-16le')
+      expect(tmx.import[1][2][3]).to eq("de-DE")
+    end
+
+    it 'imports a multiple language pair TMX file' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/multiple_language_pairs.tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
+      expect(tmx.import[1][2][3]).to eq("it")
+    end
+
+    it 'imports a srclang equals *all* TMX file' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/srclang_all.tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
+      expect(tmx.import[1][2][3]).to eq("it")
+    end
+
+    it 'imports a TMX file with out of order segments' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/out_of_order_segments.tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
+      expect(tmx.import[1][2][3]).to eq("en-US")
+    end
+
+    it 'imports a TMX file with out of order segments' do
+      file_path = File.expand_path('../tmx_importer/spec/test_sample_files/out_of_order_segments.tmx')
+      tmx = TmxImporter::Tmx.new(file_path: file_path, encoding: 'utf-8')
+      expect(tmx.import[1][2][1]).to eq("target")
+    end
+  end
+end

data/tmx_importer.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'tmx_importer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "tmx_importer"
+  spec.version       = TmxImporter::VERSION
+  spec.authors       = ["Kevin S. Dias"]
+  spec.email         = ["diasks2@gmail.com"]
+
+  spec.summary       = %q{TMX translation memory file importer}
+  spec.description   = %q{Import the content of a TMX translation memory file to your database}
+  spec.homepage      = "https://github.com/diasks2/tmx_importer"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_runtime_dependency "libxml-ruby"
+  spec.add_runtime_dependency "pretty_strings", "~> 0.5.0"
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: tmx_importer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kevin S. Dias
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: libxml-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pretty_strings
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+description: Import the content of a TMX translation memory file to your database
+email:
+- diasks2@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/tmx_importer.rb
+- lib/tmx_importer/version.rb
+- spec/spec_helper.rb
+- spec/test_sample_files/bad_encoding.tmx
+- spec/test_sample_files/bad_markup(utf-16).tmx
+- spec/test_sample_files/bad_markup(utf-8).tmx
+- spec/test_sample_files/multiple_language_pairs.tmx
+- spec/test_sample_files/out_of_order_segments.tmx
+- spec/test_sample_files/srclang_all.tmx
+- spec/test_sample_files/test_tm(utf-16LE BOM).tmx
+- spec/test_sample_files/test_tm(utf-16LE).tmx
+- spec/test_sample_files/test_tm(utf-8).tmx
+- spec/test_sample_files/test_tm_2(utf-8).tmx
+- spec/tmx_importer_spec.rb
+- tmx_importer.gemspec
+homepage: https://github.com/diasks2/tmx_importer
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: TMX translation memory file importer
+test_files:
+- spec/spec_helper.rb
+- spec/test_sample_files/bad_encoding.tmx
+- spec/test_sample_files/bad_markup(utf-16).tmx
+- spec/test_sample_files/bad_markup(utf-8).tmx
+- spec/test_sample_files/multiple_language_pairs.tmx
+- spec/test_sample_files/out_of_order_segments.tmx
+- spec/test_sample_files/srclang_all.tmx
+- spec/test_sample_files/test_tm(utf-16LE BOM).tmx
+- spec/test_sample_files/test_tm(utf-16LE).tmx
+- spec/test_sample_files/test_tm(utf-8).tmx
+- spec/test_sample_files/test_tm_2(utf-8).tmx
+- spec/tmx_importer_spec.rb