tbx_importer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4f732759e6d8b7243d4ee79cc9926309cdda6ea1
+  data.tar.gz: 27f497202083e911be6c9ab6c8aa2ccec4d977f5
+SHA512:
+  metadata.gz: 2c5e2f90baded7436b56c8f522b455cb4b696f857579376cdd51b8bfcff8ea300f395f47b054ed32c04e58bc7812e6e752cee63cde6815d68d1c13d4ce978580
+  data.tar.gz: 30e08a3ab20b63d3e3459a5f73d0d05cb69c3490b03f46e2f5f64e10345b1ff0950b2c63996b4566233d9590d89da9fa77cc64760555cd95fb66b291b742bc43

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.2.4

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tbx_importer.gemspec
+gemspec

data/README.md

@@ -0,0 +1,71 @@
+# TBX (TermBase eXchange) Importer
+
+[![Gem Version](https://badge.fury.io/rb/tbx_importer.svg)](https://badge.fury.io/rb/tbx_importer) [![Build Status](https://travis-ci.org/diasks2/tbx_importer.png)](https://travis-ci.org/diasks2/tbx_importer) [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/diasks2/tbx_importer/blob/master/LICENSE.txt)
+
+This gem handles the importing and parsing of [.tbx files](http://www.ttt.org/oscarStandards/tbx/tbx_oscar.pdf). [TMX files](http://www.ttt.org/tbx/) are xml files.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+**Ruby**  
+```
+gem install tbx_importer
+```
+
+**Ruby on Rails**  
+Add this line to your application’s Gemfile:  
+```ruby 
+gem 'tbx_importer'
+```
+
+## Usage
+
+```ruby
+# Get the high level stats of a TBX file
+# Including the encoding is optional. If not included the gem will attempt to detect the encoding.
+file_path = File.expand_path('../tbx_importer/spec/sample_files/sample.tbx')
+tbx = TbxImporter::Tbx.new(file_path: file_path)
+tbx.stats
+# => {:tc_count=>1, :term_count=>3, :language_pairs=>[["en", "fr"], ["en", "es"]]}
+
+# Extract the segments of a TBX file
+# Result: [term concepts, terms]
+# term concepts = [tu_id, definition]
+# terms = [tu_id, language, part_of_speech, term]
+
+tbx.import
+# => [[["6234-1457917153-1"], "the earth, together with all of its countries, peoples, and natural features.""], [["6234-1457917153-1", "en", "noun", world"], ["6234-1457917153-1", "fr", "noun", "monde"], ["6234-1457917153-1", "es", "noun", "mundo"]]]
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/diasks2/tbx_importer/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## License
+
+The MIT License (MIT)
+
+Copyright (c) 2016 Kevin S. Dias
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/tbx_importer/version.rb

@@ -0,0 +1,3 @@
+module TbxImporter
+  VERSION = "0.1.0"
+end

data/lib/tbx_importer.rb

@@ -0,0 +1,150 @@
+require 'tbx_importer/version'
+require 'xml'
+require 'open-uri'
+require 'pretty_strings'
+require 'charlock_holmes'
+
+module TbxImporter
+  class Tbx
+    attr_reader :file_path, :encoding
+    def initialize(file_path:, **args)
+      @file_path = file_path
+      @content = File.read(open(@file_path)) if !args[:encoding].eql?('UTF-8')
+      if args[:encoding].nil?
+        @encoding = CharlockHolmes::EncodingDetector.detect(@content[0..100_000])[:encoding]
+        if @encoding.nil?
+          encoding_in_file = @content.dup.force_encoding('utf-8').scrub!("*").gsub!(/\0/, '').scan(/(?<=encoding=").*(?=")/)[0].upcase
+          if encoding_in_file.eql?('UTF-8')
+            @encoding = ('UTF-8')
+          elsif encoding_in_file.eql?('UTF-16')
+            @encoding = ('UTF-16LE')
+          end
+        end
+      else
+        @encoding = args[:encoding].upcase
+      end
+      @doc = {
+        source_language: "",
+        tc: { id: "", counter: 0, vals: [], lang: "", definition: "" },
+        term: { lang: "", counter: 0, vals: [], part_of_speech: "" },
+        language_pairs: [],
+        term_entry: false
+      }
+      raise "Encoding type could not be determined. Please set an encoding of UTF-8, UTF-16LE, or UTF-16BE" if @encoding.nil?
+      raise "Encoding type not supported. Please choose an encoding of UTF-8, UTF-16LE, or UTF-16BE" unless @encoding.eql?('UTF-8') || @encoding.eql?('UTF-16LE') || @encoding.eql?('UTF-16BE')
+      @text = CharlockHolmes::Converter.convert(@content, @encoding, 'UTF-8') if !@encoding.eql?('UTF-8')
+    end
+
+    def stats
+      if encoding.eql?('UTF-8')
+        analyze_stats_utf_8
+      else
+        analyze_stats_utf_16
+      end
+      {tc_count: @doc[:tc][:counter], term_count: @doc[:term][:counter], language_pairs: @doc[:language_pairs].uniq}
+    end
+
+    def import
+      reader = read_file
+      parse_file(reader)
+      [@doc[:tc][:vals], @doc[:term][:vals]]
+    end
+
+    private
+
+    def analyze_stats_utf_8
+      File.readlines(@file_path).each do |line|
+        analyze_line(line)
+      end
+    end
+
+    def analyze_stats_utf_16
+      @text.each_line do |line|
+        analyze_line(line)
+      end
+    end
+
+    def read_file
+      if encoding.eql?('UTF-8')
+        XML::Reader.io(open(file_path), options: XML::Parser::Options::NOERROR, encoding: XML::Encoding::UTF_8)
+      else
+        reader = @text.gsub(/(?<=encoding=").*(?=")/, 'utf-8').gsub(/&#x[0-1]?[0-9a-fA-F];/, ' ').gsub(/[\0-\x1f\x7f\u2028]/, ' ')
+        XML::Reader.string(reader, options: XML::Parser::Options::NOERROR, encoding: XML::Encoding::UTF_8)
+      end
+    end
+
+    def analyze_line(line)
+      @doc[:term_entry] = true if !line.scan(/termEntry/).nil? && !line.scan(/termEntry/).empty?
+      if line.scan(/(?<=lang=\S)\S+(?=')/).nil? || line.scan(/(?<=lang=\S)\S+(?=')/).empty?
+        language = line.scan(/(?<=lang=\S)\S+(?=")/).uniq
+      else
+        language = line.scan(/(?<=lang=\S)\S+(?=')/).uniq
+      end
+      @doc[:source_language] = language[0] if line.include?('lang=') && !language.empty? && @doc[:source_language].empty? && @doc[:term_entry]
+      @doc[:tc][:counter] += line.scan(/<\/termEntry>/).count
+      @doc[:term][:counter] += line.scan(/<\/term>/).count
+      if !@doc[:source_language].empty?
+        language.each_with_index do |lang, index|
+          next if @doc[:source_language].eql?(lang)
+          @doc[:language_pairs] << [@doc[:source_language], lang]
+        end
+        @doc[:language_pairs] = @doc[:language_pairs].uniq
+      end
+    end
+
+    def parse_file(reader)
+      tag_stack = []
+      generate_unique_id
+      while reader.read do
+        if !tag_stack.include?(reader.name)
+          tag_stack.push(reader.name)
+          eval_state(tag_stack, reader)
+        elsif tag_stack.last == reader.name
+          if tag_stack.pop.bytes.to_a == [116, 101, 114, 109, 69, 110, 116, 114, 121]
+            generate_unique_id
+          end
+        end
+      end
+      reader.close
+    end
+
+    def eval_state(tag_stack,reader)
+      case tag_stack.last.bytes.to_a
+      when [109, 97, 114, 116, 105, 102] #martif
+        @doc[:lang] = reader.get_attribute("lang") || reader.get_attribute("xml:lang")
+        @doc[:language_pairs] << @doc[:lang]
+      when [116, 101, 114, 109, 69, 110, 116, 114, 121] #termEntry
+        write_tc
+      when [108, 97, 110, 103, 83, 101, 116] #langSet
+        @doc[:term][:lang] = reader.get_attribute("lang") || reader.get_attribute("xml:lang")
+        @doc[:language_pairs] << @doc[:term][:lang]
+      when [116, 101, 114, 109] #term
+        write_term(reader)
+      when [116, 101, 114, 109, 78, 111, 116, 101] #termNote
+        @doc[:term][:part_of_speech] = PrettyStrings::Cleaner.new(reader.read_string.downcase).pretty.gsub("\\","&#92;").gsub("'",%q(\\\')) if reader.get_attribute("type").eql?("partOfSpeech")
+        @doc[:term][:vals].pop
+        write_term(reader)
+      when [100, 101, 115, 99, 114, 105, 112] #descrip
+        @doc[:tc][:definition] = PrettyStrings::Cleaner.new(reader.read_string).pretty.gsub("\\","&#92;").gsub("'",%q(\\\')) if reader.get_attribute("type").eql?("definition")
+        @doc[:tc][:vals].pop
+        write_tc
+      end
+    end
+
+    def write_tc
+      @doc[:tc][:vals] << [@doc[:tc][:id], @doc[:tc][:definition]]
+      @doc[:tc][:definition] = ""
+    end
+
+    def write_term(reader)
+      return if reader.read_string.nil?
+      text = PrettyStrings::Cleaner.new(reader.read_string).pretty.gsub("\\","&#92;").gsub("'",%q(\\\'))
+      word_count = text.gsub("\s+", ' ').split(' ').length
+      @doc[:term][:vals] << [@doc[:tc][:id], @doc[:term][:lang], @doc[:term][:part_of_speech], text]
+    end
+
+    def generate_unique_id
+      @doc[:tc][:id] = [(1..4).map{rand(10)}.join(''), Time.now.to_i, @doc[:tc][:counter] += 1 ].join("-")
+    end
+  end
+end

data/spec/sample_files/sample_1(utf-8).tbx

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?><martif type="TBX" xml:lang="en-US">
+<martifHeader>
+  <fileDesc>
+    <titleStmt>
+      <title>Microsoft Terminology Collection Export</title>
+    </titleStmt>
+    <sourceDesc><p>Microsoft Terminology Collection</p></sourceDesc>
+  </fileDesc>
+</martifHeader>
+<text>
+  <body>
+    <termEntry id="2_76365">
+      <langSet xml:lang="en-US">
+        <descripGrp>
+          <descrip type="definition">To terminate a session with a computer accessed through a communications line usually a computer that is both distant and open to many users.</descrip>
+        </descripGrp>
+        <ntig>
+          <termGrp>
+            <term id="76365">log off</term>
+            <termNote type="partOfSpeech">Verb</termNote>
+          </termGrp>
+        </ntig>
+      </langSet>
+      <langSet xml:lang="fr-fr">
+        <ntig>
+          <termGrp>
+            <term id="215541">fermer une session</term>
+            <termNote type="partOfSpeech">Verb</termNote>
+          </termGrp>
+        </ntig>
+      </langSet>
+    </termEntry>
+    <termEntry id="3_106184"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">To shorten the fractional part of a number, increasing the last remaining (rightmost) digit or not, according to whether the deleted portion was over or under five.</descrip></descripGrp><ntig><termGrp><term id="106184">round</term><termNote type="partOfSpeech">Verb</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="106187">arrondir</term><termNote type="partOfSpeech">Verb</termNote></termGrp></ntig></langSet></termEntry><termEntry id="4_156085"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">Personally identifiable information (PII) that is protected in special ways by law or policy.</descrip></descripGrp><ntig><termGrp><term id="156085">sensitive data</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="156096">données sensibles</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="5_182425"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A business unit that is immediately under another business unit in the business hierarchy of an organization.</descrip></descripGrp><ntig><termGrp><term id="182425">child business unit</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="182429">sous-division</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="7_96886"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">The mode of a command-line application where it does not display confirmation messages or any other user interface items that normally appear on screen. The switch for quiet mode is typically '/q'.</descrip></descripGrp><ntig><termGrp><term id="96886">quiet mode</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="212192">mode silencieux</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="9_179350"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A virtual hard disk that points to and uses an entire physical disk for the purpose of converting a data disk to a virtual hard disk. You cannot turn on a virtual machine if a linked disk is attached to the virtual machine.</descrip></descripGrp><ntig><termGrp><term id="179350">linked virtual hard disk</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="179352">disque dur virtuel lié</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="10_307453"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">An interface that is used to navigate the menus of a Unified Messaging (UM) system using DTMF or touchtone inputs.</descrip></descripGrp><ntig><termGrp><term id="307453">Telephone User Interface</term><termNote type="partOfSpeech">Proper Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="318298">interface utilisateur de téléphonie</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="10_307467"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">An interface that is used to navigate the menus of a Unified Messaging (UM) system using DTMF or touchtone inputs.</descrip></descripGrp><ntig><termGrp><term id="307467">TUI</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="318299">TUI</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="10_1218391"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">An interface that is used to navigate the menus of a Unified Messaging (UM) system using DTMF or touchtone inputs.</descrip></descripGrp><ntig><termGrp><term id="1218391">touchtone interface</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="1783210">interface à tonalité</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry>
+    <termEntry id="12_233129">
+      <langSet xml:lang="en-US">
+        <descripGrp>
+          <descrip type="definition">The part of a service order that specifies detailed information about the requested service.</descrip>
+        </descripGrp>
+        <ntig>
+          <termGrp>
+            <term id="233129">service order line</term>
+            <termNote type="partOfSpeech">Noun</termNote>
+          </termGrp></ntig>
+        </langSet>
+        <langSet xml:lang="fr-fr">
+          <ntig>
+            <termGrp>
+              <term id="324824">ligne d'ordre de service</term>
+              <termNote type="partOfSpeech">Noun</termNote>
+            </termGrp>
+          </ntig>
+          <ntig>
+            <termGrp>
+              <term id="1950253">ligne commande service</term>
+              <termNote type="partOfSpeech">Noun</termNote>
+            </termGrp>
+          </ntig>
+        </langSet>
+      </termEntry><termEntry id="15_397203"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A logical grouping of managed folders. When a managed folder mailbox policy is applied to a user’s mailbox, all the managed folders that are linked to the policy are deployed in a single operation, thereby making the deployment of messaging records management (MRM) easier.</descrip></descripGrp><ntig><termGrp><term id="397203">managed folder mailbox policy</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="410598">Stratégie de boîte aux lettres de dossier géré</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="16_494929"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">The space between the inside edge of a child element and its content.</descrip></descripGrp><ntig><termGrp><term id="494929">padding</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="499409">marge intérieure</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="20_103799"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">For failover clusters or server clusters, a physical or logical entity that is capable of being managed by a cluster, brought online and taken offline, and moved between nodes. A resource can be owned only by a single node at any point in time.</descrip></descripGrp><ntig><termGrp><term id="103799">resource</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="103802">ressource</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="21_216347"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A field that can be used in a workflow rule or template as a placeholder for specific values or text in an e-mail. When you send an e-mail, these placeholders are replaced with the data that meet the conditions that have been set in the workflow rule. For example, if you send an e-mail to an account holder about a contract cancellation date, you can use a dynamic data field to automatically find and insert the correct cancellation date for that contract.</descrip></descripGrp><ntig><termGrp><term id="216347">dynamic data field</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="218231">champ de données dynamiques</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="22_364026"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A component of the World Wide Web Publishing Service (WWW service) in IIS that is responsible for configuration, by means of the metabase, and for worker process management.</descrip></descripGrp><ntig><termGrp><term id="364026">WWW Service Administration and Monitoring component</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="365290">composant Analyse et administration du service WWW</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="23_461846"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A tool on the main Windows SteadyState interface used to set schedules for software and operating system updates.</descrip></descripGrp><ntig><termGrp><term id="461846">Schedule Important Software Updates</term><termNote type="partOfSpeech">Proper Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="514693">Planification de mises à jour logicielles importantes</term><termNote type="partOfSpeech">Proper Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="24_537631"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A component that can be registered as part of the ASP.NET request lifecycle and that can read or change the request or response as it is processed. HttpModules are often used to perform special tasks that need to monitor each request, such as security or site statistics.</descrip></descripGrp><ntig><termGrp><term id="537631">HTTP module</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="541917">module HTTP</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="25_42999"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">In communications, the portion of a Data Terminal Equipment (DTE) device that sends data.</descrip></descripGrp><ntig><termGrp><term id="42999">data source</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="43001">source de données</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="26_147281"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">To deliver data to a client without a client request for the data.</descrip></descripGrp><ntig><termGrp><term id="147281">push</term><termNote type="partOfSpeech">Verb</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="147288">effectuer une transmission de type push</term><termNote type="partOfSpeech">Verb</termNote></termGrp></ntig></langSet></termEntry><termEntry id="27_136378"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A transmission medium designed for high-speed data transfers over long distances. Cable modem services and Digital Subscriber Line (DSL) are examples of broadband networks.</descrip></descripGrp><ntig><termGrp><term id="136378">broadband</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="212456">haut débit</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="27_216700"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A transmission medium designed for high-speed data transfers over long distances. Cable modem services and Digital Subscriber Line (DSL) are examples of broadband networks.</descrip></descripGrp><ntig><termGrp><term id="216700">broadband network</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="216717">réseau haut débit</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="29_45263"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A mark placed over, under, or through a character, usually to indicate a change in phonetic value from the unmarked state.</descrip></descripGrp><ntig><termGrp><term id="45263">diacritical mark</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="45266">signe diacritique</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="29_45264"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A mark placed over, under, or through a character, usually to indicate a change in phonetic value from the unmarked state.</descrip></descripGrp><ntig><termGrp><term id="45264">diacritic</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="45266">signe diacritique</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="30_237215"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">The part of a scenario that determines its uniqueness from other scenarios. </descrip></descripGrp><ntig><termGrp><term id="237215">differentiating factor</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="276593">facteur de différenciation</term><termNote type="partOfSpeech">Noun</termNote></termGrp></ntig></langSet></termEntry><termEntry id="31_1167251"><langSet xml:lang="en-US"><descripGrp><descrip type="definition">A shared service in Windows SharePoint Services and SharePoint Server that provides a means for storing, securing, and administering external content types and related objects. </descrip></descripGrp><ntig><termGrp><term id="1167251">Business Data Connectivity service</term><termNote type="partOfSpeech">Proper Noun</termNote></termGrp></ntig></langSet><langSet xml:lang="fr-fr"><ntig><termGrp><term id="1167718">Service BDC</term><termNote type="partOfSpeech">Proper Noun</termNote></termGrp></ntig></langSet></termEntry></body></text></martif>

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'tbx_importer'

data/spec/tbx_importer_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+
+describe TbxImporter do
+  it 'has a version number' do
+    expect(TbxImporter::VERSION).not_to be nil
+  end
+
+  describe '#stats' do
+    it 'reports the stats of a UTF-8 TBX file' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-8).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path)
+      expect(tbx.stats).to eq({:tc_count=>25, :term_count=>51, :language_pairs=>[["en-US", "fr-fr"]]})
+    end
+
+    it 'reports the stats of a UTF-16 TBX file' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-16).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path)
+      expect(tbx.stats).to eq({:tc_count=>10, :term_count=>21, :language_pairs=>[["EN-US", "PL"], ['EN-US', 'FR']]})
+    end
+  end
+
+  describe '#import' do
+    it 'imports a UTF-8 TBX file' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-8).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path).import
+      expect(tbx[0].length).to eq(25)
+    end
+
+    it 'imports a UTF-8 TBX file 2' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-8).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path).import
+      expect(tbx[1].length).to eq(51)
+    end
+
+    it 'imports a UTF-8 TBX file 3' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-8).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path).import
+      expect(tbx[0][-1][0]).to eq(tbx[1][-1][0])
+    end
+
+    it 'imports a UTF-8 TBX file 4' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-8).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path).import
+      expect(tbx[0][1][1]).to eq("To shorten the fractional part of a number, increasing the last remaining (rightmost) digit or not, according to whether the deleted portion was over or under five.")
+    end
+
+    it 'imports a UTF-8 TBX file 5' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-8).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path).import
+      expect(tbx[1][1][2]).to eq("verb")
+    end
+
+    it 'imports a UTF-16 TBX file' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-16).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path).import
+      expect(tbx[0].length).to eq(10)
+    end
+
+    it 'imports a UTF-16 TBX file 2' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-16).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path).import
+      expect(tbx[1].length).to eq(21)
+    end
+
+    it 'imports a UTF-16 TBX file 3' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-16).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path).import
+      expect(tbx[0][-1][0]).to eq(tbx[1][-1][0])
+    end
+
+    it 'imports a UTF-16 TBX file 4' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-16).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path).import
+      expect(tbx[0][1][1]).to eq('')
+    end
+
+    it 'imports a UTF-16 TBX file 5' do
+      file_path = File.expand_path('../tbx_importer/spec/sample_files/sample_1(utf-16).tbx')
+      tbx = TbxImporter::Tbx.new(file_path: file_path).import
+      expect(tbx[1][1][2]).to eq('')
+    end
+  end
+end

data/tbx_importer.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'tbx_importer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "tbx_importer"
+  spec.version       = TbxImporter::VERSION
+  spec.authors       = ["Kevin S. Dias"]
+  spec.email         = ["diasks2@gmail.com"]
+
+  spec.summary       = %q{TBX (TermBase eXchange) file importer}
+  spec.description   = %q{Import the content of a TBX (TermBase eXchange) file}
+  spec.homepage      = "https://github.com/diasks2/tbx_importer"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_runtime_dependency "libxml-ruby"
+  spec.add_runtime_dependency "pretty_strings", "~> 0.7.0"
+  spec.add_runtime_dependency "charlock_holmes_bundle_icu", "~> 0.6.9.2"
+end

metadata

@@ -0,0 +1,144 @@
+--- !ruby/object:Gem::Specification
+name: tbx_importer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kevin S. Dias
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: libxml-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pretty_strings
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes_bundle_icu
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.9.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.9.2
+description: Import the content of a TBX (TermBase eXchange) file
+email:
+- diasks2@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- lib/tbx_importer.rb
+- lib/tbx_importer/version.rb
+- spec/sample_files/sample_1(utf-16).tbx
+- spec/sample_files/sample_1(utf-8).tbx
+- spec/spec_helper.rb
+- spec/tbx_importer_spec.rb
+- tbx_importer.gemspec
+homepage: https://github.com/diasks2/tbx_importer
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: TBX (TermBase eXchange) file importer
+test_files:
+- spec/sample_files/sample_1(utf-16).tbx
+- spec/sample_files/sample_1(utf-8).tbx
+- spec/spec_helper.rb
+- spec/tbx_importer_spec.rb