RubyGems - tmx-parser - Versions diffs - 1.0.0 - Mend

tmx-parser 1.0.0

Files changed (16) hide show

checksums.yaml +7 -0
data/Gemfile +4 -0
data/History.txt +3 -0
data/README.md +113 -0
data/Rakefile +20 -0
data/lib/tmx-parser.rb +20 -0
data/lib/tmx-parser/document.rb +26 -0
data/lib/tmx-parser/elements.rb +85 -0
data/lib/tmx-parser/listener.rb +80 -0
data/lib/tmx-parser/sax_document.rb +73 -0
data/lib/tmx-parser/tag_names.rb +13 -0
data/lib/tmx-parser/version.rb +5 -0
data/spec/spec_helper.rb +8 -0
data/spec/tmx-parser_spec.rb +199 -0
data/tmx-parser.gemspec +24 -0
metadata +100 -0

checksums.yaml ADDED Viewed

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a95740797f7386113397902ddd8b2293066e1929
+  data.tar.gz: ad8ce82161502d6e3746889cc13e66d11869b0b6
+SHA512:
+  metadata.gz: 275c81a33ea76c4cac5e4d17e20d6b5478820991375bb739266ae820bfc52a142a83bc2d0712b823df4b7c8e2e381d6b11d71f2737a9dbd4fef8c65a39880d3c
+  data.tar.gz: 549f44444e7746edb4d3140294d73bfae5cb29d27010103966470777f570e0df6966cc1296e7ff9e62f1f47a24db1fb2361805eaecd8947c32e89b6a2abaab6d

data/Gemfile ADDED Viewed

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+gemspec

data/History.txt ADDED Viewed

@@ -0,0 +1,3 @@
+== 1.0.0
+* Birthday!

data/README.md ADDED Viewed

@@ -0,0 +1,113 @@
+tmx-parser
+=================
+Parser for the Translation Memory eXchange (.tmx) file format.
+## Installation
+`gem install tmx-parser`
+## Usage
+```ruby
+require 'tmx-parser'
+```
+## Functionality
+Got a .tmx file you need to parse? Just use the `TmxParser#load` method. It'll return an enumerable `TmxParser::Document` object for your iterating pleasure:
+```ruby
+doc = TmxParser.load(File.open('path/to/my.tmx'))
+doc.each do |unit|
+  ...
+end
+```
+You can also pass a string to `#load`:
+```ruby
+doc = TmxParser.load(File.read('path/to/my.tmx'))
+```
+The parser works in a streaming fashion, meaning it tries not to hold the entire source document in memory all at once. It will instead yield each translation unit incrementally.
+## Translation Units
+Translation units are simple Ruby objects that contain properties (tmx `<prop>` elements) and variants (tmx `tuv` elements). You can also retrieve the tuid (translation unit id) and segtype (segment type). Given this document:
+```xml
+<tmx version="1.4">
+  <body>
+    <tu tuid="79b371014a8382a3b6efb86ec6ea97d9" segtype="block">
+      <prop type="x-segment-id">0</prop>
+      <prop type="x-some-property">six.hours</prop>
+      <tuv xml:lang="en-US"><seg>6 hours</seg></tuv>
+      <tuv xml:lang="de-DE"><seg>6 Stunden</seg></tuv>
+    </tu>
+  </body>
+</tmx>
+```
+Here's what you can do:
+```ruby
+doc.each do |unit|
+  unit.tuid     # => "79b371014a8382a3b6efb86ec6ea97d9"
+  unit.segtype  # => "block"
+  unit.properties.keys                   # => ["x-segment-id", "x-some-property"]
+  unit.properties['x-segment-id'].value  # => "0"
+  variant = unit.variants.first
+  variant.locale    # => "en-US"
+  variant.elements  # => ["6 hours"]
+end
+```
+## Placeholders
+Let's consider a different document:
+```xml
+<tmx version="1.4">
+  <body>
+    <tu tuid="#{tuid}" segtype="block">
+      <prop type="x-segment-id">0</prop>
+      <tuv xml:lang="en-US">
+        <seg><ph type="x-placeholder">{0}</ph> sessions</seg>
+      </tuv>
+      <tuv xml:lang="de-DE">
+        <seg><ph type="x-placeholder">{0}</ph> Einheiten</seg>
+      </tuv>
+    </tu>
+  </body>
+</tmx>
+```
+The placeholders will be added to the variant's `elements` array:
+```ruby
+doc.each do |unit|
+  variant = unit.variants.first
+  variant.elements  # => ["#<TmxParser::Placeholder:0x5ad5be4a @text="{0}", @type="x-placeholder">", " sessions"]
+end
+```
+Begin paired tags (tmx `bpt` elements) and end paired tags (tmx `ept` elements) are handled the same way.
+## See Also
+* TMX file format: [http://www.gala-global.org/oscarStandards/tmx/tmx14b.html](http://www.gala-global.org/oscarStandards/tmx/tmx14b.html)
+## Requirements
+No external requirements.
+## Running Tests
+`bundle exec rspec` should do the trick :)
+## Authors
+* Cameron C. Dutro: http://github.com/camertron

data/Rakefile ADDED Viewed

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+$:.unshift File.join(File.dirname(__FILE__), 'lib')
+require 'rubygems' unless ENV['NO_RUBYGEMS']
+require 'bundler'
+require 'rspec/core/rake_task'
+require 'rubygems/package_task'
+require 'tmx-parser'
+Bundler::GemHelper.install_tasks
+task :default => :spec
+desc 'Run specs'
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = './spec/**/*_spec.rb'
+end

data/lib/tmx-parser.rb ADDED Viewed

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+require 'nokogiri'
+module TmxParser
+  autoload :Document,      'tmx-parser/document'
+  autoload :SaxDocument,   'tmx-parser/sax_document'
+  autoload :Listener,      'tmx-parser/listener'
+  autoload :TagNames,      'tmx-parser/tag_names'
+  autoload :Unit,          'tmx-parser/elements'
+  autoload :PropertyValue, 'tmx-parser/elements'
+  autoload :Variant,       'tmx-parser/elements'
+  autoload :Placeholder,   'tmx-parser/elements'
+  autoload :BeginPair,     'tmx-parser/elements'
+  autoload :EndPair,       'tmx-parser/elements'
+  def self.load(string_or_file_handle)
+    Document.new(string_or_file_handle)
+  end
+end

data/lib/tmx-parser/document.rb ADDED Viewed

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+module TmxParser
+  class Document
+    include Enumerable
+    attr_reader :string_or_file_handle
+    def initialize(string_or_file_handle)
+      @string_or_file_handle = string_or_file_handle
+    end
+    def each(&block)
+      if block_given?
+        listener = Listener.new(&block)
+        document = SaxDocument.new(listener)
+        parser = Nokogiri::XML::SAX::Parser.new(document)
+        parser.parse(string_or_file_handle)
+      else
+        to_enum(__method__)
+      end
+    end
+  end
+end

data/lib/tmx-parser/elements.rb ADDED Viewed

@@ -0,0 +1,85 @@
+# encoding: UTF-8
+module TmxParser
+  class Unit
+    attr_reader :tuid, :segtype, :properties, :variants
+    def initialize(tuid, segtype)
+      @tuid = tuid
+      @segtype = segtype
+      @properties = {}
+      @variants = []
+    end
+  end
+  class PropertyValue
+    attr_accessor :value
+    def initialize
+      @value = ''
+    end
+    def receive_text(str)
+      @value << str
+    end
+  end
+  class Variant
+    attr_reader :locale
+    attr_accessor :elements
+    def initialize(locale)
+      @locale = locale
+      @elements = []
+    end
+    def receive_text(str)
+      @elements << str
+    end
+  end
+  class Placeholder
+    attr_reader :type, :text
+    attr_accessor :start, :length
+    def initialize(type)
+      @type = type
+      @text = ''
+    end
+    def receive_text(str)
+      @text << str
+    end
+  end
+  class Pair
+    attr_reader :text, :i
+    def initialize(i)
+      @i = i
+      @text = ''
+    end
+    def receive_text(str)
+      @text << str
+    end
+    def type
+      raise NotImplementedError
+    end
+  end
+  class BeginPair < Pair
+    def type
+      :begin
+    end
+  end
+  class EndPair < Pair
+    def type
+      :end
+    end
+  end
+end

data/lib/tmx-parser/listener.rb ADDED Viewed

@@ -0,0 +1,80 @@
+# encoding: UTF-8
+module TmxParser
+  class Listener
+    include TagNames
+    attr_reader :units, :proc
+    def initialize(&block)
+      @stack = []
+      @proc = block
+    end
+    def unit(tuid, segtype)
+      @current_unit = Unit.new(tuid, segtype)
+    end
+    def variant(locale)
+      variant = Variant.new(locale)
+      current_unit.variants << variant
+      stack.push(variant)
+    end
+    def property(name)
+      val = PropertyValue.new
+      current_unit.properties[name] = val
+      stack.push(val)
+    end
+    def text(str)
+      if last = stack.last
+        last.receive_text(str)
+      end
+    end
+    def done(tag_name)
+      if tag_name == UNIT_TAG
+        proc.call(current_unit)
+      else
+        if tag_name_for(stack.last) == tag_name
+          stack.pop
+        end
+      end
+    end
+    def placeholder(type)
+      placeholder = Placeholder.new(type)
+      current_unit.variants.last.elements << placeholder
+      stack.push(placeholder)
+    end
+    def begin_paired_tag(i)
+      begin_pair = BeginPair.new(i)
+      current_unit.variants.last.elements << begin_pair
+      stack.push(begin_pair)
+    end
+    def end_paired_tag(i)
+      end_pair = EndPair.new(i)
+      current_unit.variants.last.elements << end_pair
+      stack.push(end_pair)
+    end
+    private
+    def tag_name_for(obj)
+      case obj
+        when Variant       then VARIANT_TAG
+        when PropertyValue then PROPERTY_TAG
+        when Placeholder   then PLACEHOLDER_TAG
+        when BeginPair     then BEGIN_PAIRED_TAG
+        when EndPair       then END_PAIRED_TAG
+      end
+    end
+    attr_reader :current_unit, :stack
+  end
+end

data/lib/tmx-parser/sax_document.rb ADDED Viewed

@@ -0,0 +1,73 @@
+# encoding: UTF-8
+require 'nokogiri'
+module TmxParser
+  class SaxDocument < Nokogiri::XML::SAX::Document
+    include TagNames
+    attr_reader :listener
+    def initialize(listener)
+      @listener = listener
+      @capture_stack = [false]
+      @text = ''
+    end
+    def start_element(name, attrs = [])
+      case name
+        when UNIT_TAG
+          listener.unit(
+            get_attr('tuid', attrs), get_attr('segtype', attrs)
+          )
+        when VARIANT_TAG
+          locale = get_attr('xml:lang', attrs)
+          listener.variant(locale)
+        when SEGMENT_TAG
+          capture_text
+        when PROPERTY_TAG
+          capture_text
+          listener.property(get_attr('type', attrs))
+        when BEGIN_PAIRED_TAG
+          capture_text
+          listener.begin_paired_tag(get_attr('i', attrs))
+        when END_PAIRED_TAG
+          capture_text
+          listener.end_paired_tag(get_attr('i', attrs))
+        when PLACEHOLDER_TAG
+          capture_text
+          listener.placeholder(get_attr('type', attrs))
+      end
+    end
+    def end_element(name)
+      @capture_stack.pop
+      send_text
+      listener.done(name)
+    end
+    def characters(str)
+      @text += str if @capture_stack.last
+    end
+    private
+    def send_text
+      listener.text(@text) unless @text.empty?
+      @text = ''
+    end
+    def capture_text
+      send_text
+      @capture_stack.push(true)
+    end
+    def get_attr(name, attrs)
+      if found = attrs.find { |a| a.first == name }
+        found.last
+      end
+    end
+  end
+end

data/lib/tmx-parser/tag_names.rb ADDED Viewed

@@ -0,0 +1,13 @@
+# encoding: UTF-8
+module TmxParser
+  module TagNames
+    UNIT_TAG         = 'tu'
+    VARIANT_TAG      = 'tuv'
+    PROPERTY_TAG     = 'prop'
+    SEGMENT_TAG      = 'seg'
+    PLACEHOLDER_TAG  = 'ph'
+    BEGIN_PAIRED_TAG = 'bpt'
+    END_PAIRED_TAG   = 'ept'
+  end
+end

data/lib/tmx-parser/version.rb ADDED Viewed

@@ -0,0 +1,5 @@
+# encoding: UTF-8
+module TmxParser
+  VERSION = '1.0.0'
+end

data/spec/spec_helper.rb ADDED Viewed

@@ -0,0 +1,8 @@
+# encoding: UTF-8
+require 'pry-nav'
+require 'rspec'
+require 'tmx-parser'
+RSpec.configure do |config|
+end

data/spec/tmx-parser_spec.rb ADDED Viewed

@@ -0,0 +1,199 @@
+# encoding: UTF-8
+require 'spec_helper'
+describe TmxParser do
+  let(:parser) { TmxParser }
+  let(:tuid) { '79b371014a8382a3b6efb86ec6ea97d9' }
+  def find_variant(locale, unit)
+    unit.variants.find { |v| v.locale == locale }
+  end
+  context 'with a basic tmx document' do
+    let(:document) do
+      %Q{
+        <tmx version="1.4">
+          <body>
+            <tu tuid="#{tuid}" segtype="block">
+              <prop type="x-segment-id">0</prop>
+              <prop type="x-some-property">six.hours</prop>
+              <tuv xml:lang="en-US"><seg>6 hours</seg></tuv>
+              <tuv xml:lang="de-DE"><seg>6 Stunden</seg></tuv>
+            </tu>
+          </body>
+        </tmx>
+      }
+    end
+    it 'identifies the tuid and segtype' do
+      parser.load(document).to_a.tap do |units|
+        expect(units.size).to eq(1)
+        units.first.tap do |unit|
+          expect(unit.tuid).to eq(tuid)
+          expect(unit.segtype).to eq('block')
+        end
+      end
+    end
+    it 'identifies the correct variants' do
+      parser.load(document).to_a.first.tap do |unit|
+        expect(unit.variants.size).to eq(2)
+        expect(find_variant('en-US', unit).elements).to eq(['6 hours'])
+        expect(find_variant('de-DE', unit).elements).to eq(['6 Stunden'])
+        unit.variants.each do |variant|
+          expect(variant).to be_a(TmxParser::Variant)
+        end
+      end
+    end
+    it 'identifies properties' do
+      parser.load(document).to_a.first.tap do |unit|
+        expect(unit.properties.size).to eq(2)
+        expect(unit.properties).to include('x-segment-id')
+        expect(unit.properties).to include('x-some-property')
+        expect(unit.properties['x-segment-id'].value).to eq('0')
+        expect(unit.properties['x-some-property'].value).to eq('six.hours')
+      end
+    end
+  end
+  context 'with a tmx document that contains a property that makes jruby cry' do
+    # For some reason, jruby doesn't like square brackets in property values.
+    # See: https://github.com/sparklemotion/nokogiri/issues/1261
+    let(:document) do
+      %Q{
+        <tmx version="1.4">
+          <body>
+            <tu tuid="#{tuid}" segtype="block">
+              <prop type="x-segment-id">0</prop>
+              <prop type="x-some-property">en:#:daily-data:#:[3]:#:times</prop>
+              <tuv xml:lang="en-US"><seg>6 hours</seg></tuv>
+              <tuv xml:lang="de-DE"><seg>6 Stunden</seg></tuv>
+            </tu>
+          </body>
+        </tmx>
+      }
+    end
+    it 'identifies the property correctly' do
+      parser.load(document).to_a.first.tap do |unit|
+        expect(unit.properties).to include('x-some-property')
+        expect(unit.properties['x-some-property']).to be_a(TmxParser::PropertyValue)
+        expect(unit.properties['x-some-property'].value).to eq(
+          'en:#:daily-data:#:[3]:#:times'
+        )
+      end
+    end
+  end
+  context 'with a tmx document that contains placeholders' do
+    let(:document) do
+      %Q{
+        <tmx version="1.4">
+          <body>
+            <tu tuid="#{tuid}" segtype="block">
+              <prop type="x-segment-id">0</prop>
+              <tuv xml:lang="en-US">
+                <seg><ph type="x-placeholder">{0}</ph> sessions</seg>
+              </tuv>
+              <tuv xml:lang="de-DE">
+                <seg><ph type="x-placeholder">{0}</ph> Einheiten</seg>
+              </tuv>
+            </tu>
+          </body>
+        </tmx>
+      }
+    end
+    it 'identifies the placeholders' do
+      parser.load(document).to_a.first.tap do |unit|
+        expect(unit.variants.size).to eq(2)
+        find_variant('en-US', unit).tap do |en_variant|
+          expect(en_variant.elements.size).to eq(2)
+          en_variant.elements.first.tap do |first_element|
+            expect(first_element.type).to eq('x-placeholder')
+            expect(first_element.text).to eq('{0}')
+          end
+          en_variant.elements.last.tap do |last_element|
+            expect(last_element).to be_a(String)
+            expect(last_element).to eq(' sessions')
+          end
+        end
+        find_variant('de-DE', unit).tap do |en_variant|
+          expect(en_variant.elements.size).to eq(2)
+          en_variant.elements.first.tap do |first_element|
+            expect(first_element).to be_a(TmxParser::Placeholder)
+            expect(first_element.type).to eq('x-placeholder')
+            expect(first_element.text).to eq('{0}')
+          end
+          en_variant.elements.last.tap do |last_element|
+            expect(last_element).to be_a(String)
+            expect(last_element).to eq(' Einheiten')
+          end
+        end
+      end
+    end
+  end
+  context 'with a tmx document that contains paired tags' do
+    let(:document) do
+      %Q{
+        <tmx version="1.4">
+          <body>
+            <tu tuid="#{tuid}" segtype="block">
+              <prop type="x-segment-id">0</prop>
+              <tuv xml:lang="en-US">
+                <seg>Build your healthy habit of daily training with <bpt i="3">&lt;strong&gt;</bpt>email training reminders.<ept i="3">&lt;/strong&gt;</ept></seg>
+              </tuv>
+              <tuv xml:lang="de-DE">
+                <seg><bpt i="3">&lt;strong&gt;</bpt>Mit Erinnerungen per E-Mail<ept i="3">&lt;/strong&gt;</ept> können Sie das tägliche Training zu einer schönen Angewohnheit werden lassen.</seg>
+              </tuv>
+            </tu>
+          </body>
+        </tmx>
+      }
+    end
+    it 'identifies the tags' do
+      parser.load(document).to_a.first.tap do |unit|
+        expect(unit.variants.size).to eq(2)
+        find_variant('en-US', unit).tap do |en_variant|
+          expect(en_variant.elements.size).to eq(4)
+          en_variant.elements[0].tap do |element|
+            expect(element).to be_a(String)
+            expect(element).to eq('Build your healthy habit of daily training with ')
+          end
+          en_variant.elements[1].tap do |element|
+            expect(element).to be_a(TmxParser::BeginPair)
+            expect(element.i).to eq('3')
+            expect(element.text).to eq('<strong>')
+          end
+          en_variant.elements[2].tap do |element|
+            expect(element).to be_a(String)
+            expect(element).to eq('email training reminders.')
+          end
+          en_variant.elements[3].tap do |element|
+            expect(element).to be_a(TmxParser::EndPair)
+            expect(element.i).to eq('3')
+            expect(element.text).to eq('</strong>')
+          end
+        end
+      end
+    end
+  end
+end

data/tmx-parser.gemspec ADDED Viewed

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+$:.unshift File.join(File.dirname(__FILE__), 'lib')
+require 'tmx-parser/version'
+Gem::Specification.new do |s|
+  s.name     = "tmx-parser"
+  s.version  = ::TmxParser::VERSION
+  s.authors  = ["Cameron Dutro"]
+  s.email    = ["camertron@gmail.com"]
+  s.homepage = "http://github.com/camertron"
+  s.description = s.summary = "Parser for the Translation Memory eXchange (.tmx) file format."
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.require_path = 'lib'
+  s.files = Dir["{lib,spec}/**/*", "Gemfile", "History.txt", "README.md", "Rakefile", "tmx-parser.gemspec"]
+  s.add_dependency 'nokogiri', '~> 1.6.0'
+  s.add_development_dependency 'pry-nav', '~> 0.2.0'
+  s.add_development_dependency 'rspec', '~> 3.2.0'
+end

metadata ADDED Viewed

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: tmx-parser
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Cameron Dutro
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-03-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+description: Parser for the Translation Memory eXchange (.tmx) file format.
+email:
+- camertron@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- History.txt
+- README.md
+- Rakefile
+- lib/tmx-parser.rb
+- lib/tmx-parser/document.rb
+- lib/tmx-parser/elements.rb
+- lib/tmx-parser/listener.rb
+- lib/tmx-parser/sax_document.rb
+- lib/tmx-parser/tag_names.rb
+- lib/tmx-parser/version.rb
+- spec/spec_helper.rb
+- spec/tmx-parser_spec.rb
+- tmx-parser.gemspec
+homepage: http://github.com/camertron
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.6
+signing_key:
+specification_version: 4
+summary: Parser for the Translation Memory eXchange (.tmx) file format.
+test_files: []
+has_rdoc: true