txml_importer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 408bb4b21de0373f60c443d3bed90f95ccf26465
+  data.tar.gz: 27f0547d2809c1418f0ff6b1473e8bc4e56de6f1
+SHA512:
+  metadata.gz: 7c0ef7a29f39360b50541b6af82a7d1ec1d7824ae1bded610d4358d9fe57657341b9dc9289a417c790987955408b7ae77207736ba17e95b72f91c392719d9a30
+  data.tar.gz: 426d75dc482a762b15506a3ce92205e3ed80c7f92bb53541f76147994f0006211a5233cc50965f988671b7539b861a87775aa0df9c5734abca33bfefd29cfb96

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.2.4

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in txml_importer.gemspec
+gemspec

data/README.md

@@ -0,0 +1,70 @@
+# TXML Importer
+
+[![Gem Version](https://badge.fury.io/rb/txml_importer.svg)](https://badge.fury.io/rb/txml_importer) [![Build Status](https://travis-ci.org/diasks2/txml_importer.png)](https://travis-ci.org/diasks2/txml_importer) [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/diasks2/txml_importer/blob/master/LICENSE.txt)
+
+This gem handles the importing and parsing of .txml translation memory files.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+**Ruby**  
+```
+gem install txml_importer
+```
+
+**Ruby on Rails**  
+Add this line to your application’s Gemfile:  
+```ruby 
+gem 'txml_importer'
+```
+
+## Usage
+
+```ruby
+# Get the high level stats of a .txml file
+# Including the encoding is optional. If not included the gem will attempt to detect the encoding.
+file_path = File.expand_path('../txml_importer/spec/test_sample_files/sample.txml')
+TxmlImporter::Txml.new(file_path: file_path).stats
+# => {:tu_count=>1, :seg_count=>2, :language_pairs=>[["en", "fr"]]}
+
+# Extract the segments of a .txml file
+# Result: [translation_units, segments]
+# translation_units = [tu_id]
+# segments = [tu_id, segment_role, word_count, language, segment_text]
+
+TxmlImporter::Txml.new(file_path: file_path).import
+# => [[["6234-1457917153-1"]], [["6234-1457917153-1", "source", 2, "en", "Hello world"], ["6234-1457917153-1", "target", 3, "fr", "Bonjour le monde"]]]
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/diasks2/txml_importer/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## License
+
+The MIT License (MIT)
+
+Copyright (c) 2016 Kevin S. Dias
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,5 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+task default: :spec

data/lib/txml_importer/version.rb

@@ -0,0 +1,3 @@
+module TxmlImporter
+  VERSION = "0.1.0"
+end

data/lib/txml_importer.rb

@@ -0,0 +1,129 @@
+require 'txml_importer/version'
+require 'xml'
+require 'open-uri'
+require 'pretty_strings'
+require 'charlock_holmes'
+
+module TxmlImporter
+  class Txml
+    attr_reader :file_path, :encoding
+    def initialize(file_path:, **args)
+      @file_path = file_path
+      @content = File.read(open(@file_path)) if !args[:encoding].eql?('UTF-8')
+      if args[:encoding].nil?
+        @encoding = CharlockHolmes::EncodingDetector.detect(@content[0..100_000])[:encoding]
+        if @encoding.nil?
+          encoding_in_file = @content.dup.force_encoding('utf-8').scrub!("*").gsub!(/\0/, '').scan(/(?<=encoding=").*(?=")/)[0].upcase
+          if encoding_in_file.eql?('UTF-8')
+            @encoding = ('UTF-8')
+          elsif encoding_in_file.eql?('UTF-16')
+            @encoding = ('UTF-16LE')
+          end
+        end
+      else
+        @encoding = args[:encoding].upcase
+      end
+      @doc = {
+        source_language: "",
+        tu: { id: "", counter: 0, vals: [] },
+        seg: { counter: 0, vals: [] },
+        language_pairs: []
+      }
+      raise "Encoding type could not be determined. Please set an encoding of UTF-8, UTF-16LE, or UTF-16BE" if @encoding.nil?
+      raise "Encoding type not supported. Please choose an encoding of UTF-8, UTF-16LE, or UTF-16BE" unless @encoding.eql?('UTF-8') || @encoding.eql?('UTF-16LE') || @encoding.eql?('UTF-16BE')
+      @text = CharlockHolmes::Converter.convert(@content, @encoding, 'UTF-8') if !@encoding.eql?('UTF-8')
+    end
+
+    def stats
+      if encoding.eql?('UTF-8')
+        analyze_stats_utf_8
+      else
+        analyze_stats_utf_16
+      end
+      {tu_count: @doc[:tu][:counter], seg_count: @doc[:seg][:counter], language_pairs: @doc[:language_pairs].uniq}
+    end
+
+    def import
+      reader = read_file
+      parse_file(reader)
+      [@doc[:tu][:vals], @doc[:seg][:vals]]
+    end
+
+    private
+
+    def analyze_stats_utf_8
+      File.readlines(@file_path).each do |line|
+        analyze_line(line)
+      end
+    end
+
+    def analyze_stats_utf_16
+      @text.each_line do |line|
+        analyze_line(line)
+      end
+    end
+
+    def read_file
+      if encoding.eql?('UTF-8')
+        XML::Reader.io(open(file_path), options: XML::Parser::Options::NOERROR, encoding: XML::Encoding::UTF_8)
+      else
+        reader = @text.gsub(/(?<=encoding=").*(?=")/, 'utf-8').gsub(/&#x[0-1]?[0-9a-fA-F];/, ' ').gsub(/[\0-\x1f\x7f\u2028]/, ' ')
+        XML::Reader.string(reader, options: XML::Parser::Options::NOERROR, encoding: XML::Encoding::UTF_8)
+      end
+    end
+
+    def analyze_line(line)
+      @doc[:source_language] = line.scan(/(?<=locale=\S)\S+(?=")/)[0] if line.include?('locale=') && !line.scan(/(?<=locale=\S)\S+(?=")/).empty?
+      @doc[:source_language] = line.scan(/(?<=locale=\S)\S+(?=')/)[0] if line.include?('locale=') && !line.scan(/(?<=locale=\S)\S+(?=')/).empty?
+      @doc[:target_language] = line.scan(/(?<=targetlocale=\S)\S+(?=")/)[0] if line.include?('targetlocale=') && !line.scan(/(?<=targetlocale=\S)\S+(?=")/).empty?
+      @doc[:target_language] = line.scan(/(?<=targetlocale=\S)\S+(?=')/)[0] if line.include?('targetlocale=') && !line.scan(/(?<=targetlocale=\S)\S+(?=')/).empty?
+      @doc[:tu][:counter] += line.scan(/<\/segment>/).count
+      @doc[:seg][:counter] += (line.scan(/<\/source>/).count + line.scan(/<\/target>/).count - line.scan(/<\/target><\/revision>/).count)
+      @doc[:language_pairs] << [@doc[:source_language], @doc[:target_language]] if !@doc[:source_language].empty? && !@doc[:source_language].nil? && !@doc[:target_language].empty? && !@doc[:target_language].nil?
+    end
+
+    def parse_file(reader)
+      last_tag = ''
+      @count = 0
+      while reader.read do
+        unless last_tag.bytes.to_a.eql?([114, 101, 118, 105, 115, 105, 111, 110])
+          case reader.name.bytes.to_a
+          when [116, 120, 109, 108]
+            @doc[:source_language] = reader.get_attribute("locale") if reader.has_attributes? && reader.get_attribute("locale")
+            @doc[:target_language] = reader.get_attribute("targetlocale") if reader.has_attributes? && reader.get_attribute("targetlocale")
+          when [115, 101, 103, 109, 101, 110, 116]
+            generate_unique_id if @count % 2 == 0
+            write_tu(reader) if @count % 2 == 0
+            @count += 1
+          when [115, 111, 117, 114, 99, 101]
+            write_seg(reader, 'source')
+          when [116, 97, 114, 103, 101, 116]
+            write_seg(reader, 'target')
+          end
+        end
+        last_tag = reader.name
+      end
+      reader.close
+    end
+
+    def write_tu(reader)
+      @doc[:tu][:vals] << [@doc[:tu][:id]]
+    end
+
+    def write_seg(reader, role)
+      return if reader.read_string.nil?
+      text = PrettyStrings::Cleaner.new(reader.read_string).pretty.gsub("\\","&#92;").gsub("'",%q(\\\'))
+      word_count = text.gsub("\s+", ' ').split(' ').length
+      if role.eql?('source')
+        language = @doc[:source_language]
+      else
+        language = @doc[:target_language]
+      end
+      @doc[:seg][:vals] << [@doc[:tu][:id], role, word_count, language, text]
+    end
+
+    def generate_unique_id
+      @doc[:tu][:id] = [(1..4).map{rand(10)}.join(''), Time.now.to_i, @doc[:tu][:counter] += 1 ].join("-")
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'txml_importer'

data/spec/txml_importer_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe TxmlImporter do
+  it 'has a version number' do
+    expect(TxmlImporter::VERSION).not_to be nil
+  end
+
+  describe '#stats' do
+    it 'reports the stats of a .txml file' do
+      file_path = File.expand_path('../txml_importer/spec/sample_files/sample_1.txml')
+      txml = TxmlImporter::Txml.new(file_path: file_path)
+      expect(txml.stats).to eq({:tu_count=>112, :seg_count=>224, :language_pairs=>[["FR-FR", "EN"]]})
+    end
+  end
+
+  describe '#import' do
+    it 'imports a .txml file' do
+      file_path = File.expand_path('../txml_importer/spec/sample_files/sample_1.txml')
+      txml = TxmlImporter::Txml.new(file_path: file_path).import
+      expect(txml[0].length).to eq(112)
+    end
+
+    it 'imports a .txml file' do
+      file_path = File.expand_path('../txml_importer/spec/sample_files/sample_1.txml')
+      txml = TxmlImporter::Txml.new(file_path: file_path).import
+      expect(txml[1].length).to eq(224)
+    end
+
+    it 'imports a .txml file' do
+      file_path = File.expand_path('../txml_importer/spec/sample_files/sample_1.txml')
+      txml = TxmlImporter::Txml.new(file_path: file_path).import
+      expect(txml[0][0][0]).to eq(txml[1][0][0])
+    end
+
+    it 'imports a .txml file' do
+      file_path = File.expand_path('../txml_importer/spec/sample_files/sample_1.txml')
+      txml = TxmlImporter::Txml.new(file_path: file_path).import
+      expect(txml[0][-1][0]).to eq(txml[1][-1][0])
+    end
+
+    it 'imports a .txml file' do
+      file_path = File.expand_path('../txml_importer/spec/sample_files/sample_1.txml')
+      txml = TxmlImporter::Txml.new(file_path: file_path)
+      expect(txml.import[1][-1][1]).to eq('target')
+    end
+  end
+end

data/txml_importer.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'txml_importer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "txml_importer"
+  spec.version       = TxmlImporter::VERSION
+  spec.authors       = ["Kevin S. Dias"]
+  spec.email         = ["diasks2@gmail.com"]
+
+  spec.summary       = %q{.txml file importer}
+  spec.description   = %q{Import the content of a .txml file.}
+  spec.homepage      = "https://github.com/diasks2/txml_importer"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_runtime_dependency "libxml-ruby"
+  spec.add_runtime_dependency "pretty_strings", "~> 0.5.0"
+  spec.add_runtime_dependency "charlock_holmes_bundle_icu", "~> 0.6.9.2"
+end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: txml_importer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kevin S. Dias
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: libxml-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pretty_strings
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes_bundle_icu
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.9.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.9.2
+description: Import the content of a .txml file.
+email:
+- diasks2@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- lib/txml_importer.rb
+- lib/txml_importer/version.rb
+- spec/spec_helper.rb
+- spec/txml_importer_spec.rb
+- txml_importer.gemspec
+homepage: https://github.com/diasks2/txml_importer
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: ".txml file importer"
+test_files:
+- spec/spec_helper.rb
+- spec/txml_importer_spec.rb