translatomatic 0.1.0 → 0.1.1

data/lib/translatomatic/resource_file.rb

@@ -1,74 +1,76 @@
-
-module Translatomatic
-  module ResourceFile
-    class << self
-      include Translatomatic::Util
-    end
-
-    # Load a resource file. If locale is not specified, the locale of the
-    # file will be determined from the filename, or else the current default
-    # locale will be used.
-    # @param [String] path Path to the resource file
-    # @param [String] locale Locale of the resource file
-    # @return [Translatomatic::ResourceFile::Base] The resource file, or nil
-    #   if the file type is unsupported.
-    def self.load(path, locale = nil)
-      path = path.kind_of?(Pathname) ? path : Pathname.new(path)
-      modules.each do |mod|
-        # match on entire filename to support extensions containing locales
-        if extension_match(mod, path)
-          log.debug("attempting to load #{path.to_s} using #{mod.name.demodulize}")
-          file = mod.new(path, locale)
-          return file if file.valid?
-        end
-      end
-      nil
-    end
-
-    # Find all resource files under the given directory. Follows symlinks.
-    # @param [String, Pathname] path The path to search from
-    # @return [Array<Translatomatic::ResourceFile>] Resource files found
-    def self.find(path, options = {})
-      files = []
-      include_dot_directories = options[:include_dot_directories]
-      path = Pathname.new(path) unless path.kind_of?(Pathname)
-      path.find do |file|
-        if !include_dot_directories && file.basename.to_s[0] == ?.
-          Find.prune
-        else
-          resource = load(file)
-          files << resource if resource
-        end
-      end
-      files
-    end
-
-    # Find all configured resource file classes
-    # @return [Array<Class>] Available resource file classes
-    def self.modules
-      self.constants.map { |c| self.const_get(c) }.select do |klass|
-        klass.is_a?(Class) && klass != Base
-      end
-    end
-
-    private
-
-    def self.extension_match(mod, path)
-      filename = path.basename.to_s.downcase
-      mod.extensions.each do |extension|
-        # don't match end of line in case file has locale extension
-        return true if filename.match(/\.#{extension}/)
-      end
-      false
-    end
-  end
-end
-
-require 'translatomatic/resource_file/base'
-require 'translatomatic/resource_file/yaml'
-require 'translatomatic/resource_file/properties'
-require 'translatomatic/resource_file/text'
-require 'translatomatic/resource_file/xml'
-require 'translatomatic/resource_file/html'
-require 'translatomatic/resource_file/plist'
-require 'translatomatic/resource_file/xcode_strings'
+
+module Translatomatic
+  module ResourceFile
+    class << self
+      include Translatomatic::Util
+    end
+
+    # Load a resource file. If locale is not specified, the locale of the
+    # file will be determined from the filename, or else the current default
+    # locale will be used.
+    # @param [String] path Path to the resource file
+    # @param [String] locale Locale of the resource file
+    # @return [Translatomatic::ResourceFile::Base] The resource file, or nil
+    #   if the file type is unsupported.
+    def self.load(path, locale = nil)
+      path = path.kind_of?(Pathname) ? path : Pathname.new(path)
+      modules.each do |mod|
+        # match on entire filename to support extensions containing locales
+        if extension_match(mod, path)
+          log.debug("attempting to load #{path.to_s} using #{mod.name.demodulize}")
+          file = mod.new(path, locale)
+          return file if file.valid?
+        end
+      end
+      nil
+    end
+
+    # Find all resource files under the given directory. Follows symlinks.
+    # @param [String, Pathname] path The path to search from
+    # @return [Array<Translatomatic::ResourceFile>] Resource files found
+    def self.find(path, options = {})
+      files = []
+      include_dot_directories = options[:include_dot_directories]
+      path = Pathname.new(path) unless path.kind_of?(Pathname)
+      path.find do |file|
+        if !include_dot_directories && file.basename.to_s[0] == ?.
+          Find.prune
+        else
+          resource = load(file)
+          files << resource if resource
+        end
+      end
+      files
+    end
+
+    # Find all configured resource file classes
+    # @return [Array<Class>] Available resource file classes
+    def self.modules
+      self.constants.map { |c| self.const_get(c) }.select do |klass|
+        klass.is_a?(Class) && klass != Base
+      end
+    end
+
+    private
+
+    def self.extension_match(mod, path)
+      filename = path.basename.to_s.downcase
+      mod.extensions.each do |extension|
+        # don't match end of line in case file has locale extension
+        return true if filename.match(/\.#{extension}/)
+      end
+      false
+    end
+  end
+end
+
+require 'translatomatic/resource_file/base'
+require 'translatomatic/resource_file/yaml'
+require 'translatomatic/resource_file/properties'
+require 'translatomatic/resource_file/text'
+require 'translatomatic/resource_file/xml'
+require 'translatomatic/resource_file/html'
+require 'translatomatic/resource_file/markdown'
+require 'translatomatic/resource_file/plist'
+require 'translatomatic/resource_file/resw'
+require 'translatomatic/resource_file/xcode_strings'

data/lib/translatomatic/string.rb

@@ -0,0 +1,160 @@
+module Translatomatic
+  class String
+
+    # @return [String] The string
+    attr_reader :value
+
+    # @return [Translatomatic::Locale] The string's locale
+    attr_reader :locale
+
+    # @return [Translatomatic::String] If this string is a substring of
+    #   another string, returns the original string. Otherwise, returns nil.
+    attr_reader :parent
+
+    # @return [Number] If this string is a substring of another string,
+    #   returns the starting offset of this string in the original.
+    attr_reader :offset
+
+    def initialize(value, locale, options = {})
+      @value = value || ''
+      @locale = Translatomatic::Locale.parse(locale)
+      @offset = options[:offset] || 0
+      @parent = options[:parent]
+    end
+
+    # @return [String] The value of the string
+    def to_s
+      @value
+    end
+
+    def length
+      @value.length
+    end
+
+    def empty?
+      @value.empty?
+    end
+
+    def match(regex)
+      @value.match(regex)
+    end
+
+    # @return [boolean] true if this string is a substring of another string
+    def substring?
+      @parent ? true : false
+    end
+
+    # @return [Symbol] The type of string, corresponding to TMX segtype.
+    # @see http://xml.coverpages.org/tmxSpec971212.html#SEGTYPE
+    def type
+      if sentences.length >= 2
+        :paragraph
+      else
+        script = script_data
+        @value.strip.match(/#{script.delimiter}\s*$/) ? :sentence : :phrase
+      end
+    end
+
+    # Find all sentences in the string
+    # @return [Array<Translatomatic::String] List of sentences
+    def sentences
+      sentences = @value.scan(sentence_regex)
+      strings = []
+      offset = 0
+      sentences.each do |sentence|
+        # find leading and trailing whitespace
+        next if sentence.length == 0
+
+        parts = sentence.match(/^(\s*)(.*?)(\s*)$/).to_a
+        value = parts[2]
+        offset += parts[1].length  # leading whitespace
+        strings << self.class.new(value, locale, offset: offset, parent: self)
+        offset += value.length + parts[3].length
+      end
+
+      # return [self] if there's only one sentence and it's equal to self
+      strings.length == 1 && strings[0].eql?(self) ? [self] : strings
+    end
+
+    def eql?(other)
+      other.kind_of?(Translatomatic::String) && other.hash == hash
+    end
+
+    def ==(other)
+      eql?(other)
+    end
+
+    def hash
+      [value, locale].hash
+    end
+
+    private
+
+    class Script
+      attr_reader :language
+      attr_reader :delimiter      # sentence delimiter
+      attr_reader :trailing_space # delimiter requires trailing space or eol
+      attr_reader :left_to_right  # script direction
+
+      def initialize(language:, delimiter:, trailing_space:, direction:)
+        @language = language
+        @delimiter = delimiter
+        @trailing_space = trailing_space
+        @left_to_right = direction == :ltr
+        raise "invalid direction" unless [:ltr, :rtl].include?(direction)
+      end
+    end
+
+    SCRIPT_DATA = [
+      # [language, delimiter, trailing space, direction]
+      # japanese, no space after
+      ["ja", "\u3002", false, :ltr],
+      # chinese, no space after
+      ["zh", "\u3002", false, :ltr],  # can be written any direction
+       # armenian, space after
+      ["hy", ":", true, :ltr],
+      # hindi, space after
+      ["hi", "।", true, :ltr],
+      # urdu, space after, right to left
+      ["ur", "\u06d4", true, :rtl],
+      # thai, spaces used to separate sentences
+      ["th", "\\s", false, :ltr],
+      # arabic, right to left
+      ["ar", "\\.", true, :rtl],
+      # hebrew, right to left
+      ["he", "\\.", true, :rtl],
+      # all other languages
+      ["default", "\\.", true, :ltr],
+    ]
+
+    class << self
+      attr_reader :script_data
+    end
+
+    begin
+      script_data = {}
+      SCRIPT_DATA.each do |lang, delimiter, trailing, ltr|
+        script = Script.new(language: lang, delimiter: delimiter,
+          trailing_space: trailing, direction: ltr)
+        script_data[lang] = script
+      end
+      @script_data = script_data
+    end
+
+    def sentence_regex
+      script = script_data
+      if script.trailing_space
+        regex = /.*?(?:#{script.delimiter}\s+|$)/
+      else
+        # no trailing space after delimiter
+        regex = /.*?(?:#{script.delimiter}|$)/
+      end
+    end
+
+    def script_data
+      data = self.class.script_data
+      data[locale.language] || data["default"]
+    end
+
+  end
+end

data/lib/translatomatic/tmx/document.rb

@@ -0,0 +1,100 @@
+module Translatomatic::TMX
+  # Translation Memory Exchange document
+  class Document
+
+    # Create a new instance
+    # @param [Array<TranslationUnit>] A list of translation units
+    # @param [Locale] Source locale
+    # @return A new TMX object
+    def initialize(units, source_locale, origin)
+      units = [units] unless units.kind_of?(Array)
+      @units = units
+      @source_locale = source_locale
+      @origin = origin
+    end
+
+    # @return [String] An XML string
+    def to_xml(options = {})
+      builder = Nokogiri::XML::Builder.new do |xml|
+        dtd = options[:dtd] || TMX_DTD
+        xml.doc.create_internal_subset('tmx', nil, dtd)
+        xml.tmx(version: "1.4") do
+          xml.header(creationtool: "Translatomatic",
+            creationtoolversion: Translatomatic::VERSION,
+            datatype: "PlainText",
+            segtype: "phrase",  # default segtype
+            adminlang: @source_locale.to_s,
+            srclang: @source_locale.to_s,
+            "o-tmx": @origin
+          )
+          xml.body { tmx_body(xml) }
+        end
+      end
+      builder.to_xml
+    end
+
+    # Create a TMX document from the given converter
+    # @param [Array<Translatomatic::Model::Text>] List of texts
+    # @return [Translatomatic::TMX::Document] TMX document
+    def self.from_texts(texts)
+      # group texts by from_text_id to create units
+      # source_locale: use from_text.locale
+      # origin: use text.translator
+      origins = texts.collect { |i| i.translator }.compact.uniq
+      raise "Multiple origins in texts" if origins.length > 1
+      sources = texts.select { |i| i.from_text.nil? }
+      source_locales = sources.collect { |i| i.locale }.uniq
+      raise "Multiple source locales in texts" if source_locales.length > 1
+      units = units_from_texts(texts)
+
+      return new(units, source_locales[0], origins[0])
+    end
+
+    def self.valid?(xml)
+      options = Nokogiri::XML::ParseOptions::DTDVALID
+      doc = Nokogiri::XML::Document.parse(xml, nil, nil, options)
+      doc.internal_subset.validate(doc)
+    end
+
+    private
+
+    class << self
+      include Translatomatic::Util
+    end
+
+    TMX_DTD = "http://www.ttt.org/oscarstandards/tmx/tmx14.dtd"
+
+    def tmx_body(xml)
+      @units.each do |unit|
+        xml.tu("segtype": unit.strings[0].type) do
+          unit.strings.each do |string|
+            xml.tuv("xml:lang": string.locale.to_s) do
+              xml.seg string.value
+            end
+          end
+        end
+      end
+    end
+
+    # @return [Array<Translatomatic::TMX::TranslationUnit] translation unit list
+    def self.units_from_texts(texts)
+      # group texts by from_text_id
+      texts_by_from_id = {}
+      texts.each do |text|
+        id = text.from_text_id || text.id
+        list = (texts_by_from_id[id] ||= [])
+        list << text
+      end
+
+      # create list of Translation Units
+      texts_by_from_id.values.collect do |list|
+        tmx_unit(list.uniq.collect { |i| string(i.value, i.locale) })
+      end
+    end
+
+    def self.tmx_unit(strings)
+      Translatomatic::TMX::TranslationUnit.new(strings)
+    end
+
+  end # class
+end   # module

data/lib/translatomatic/tmx/translation_unit.rb

@@ -0,0 +1,19 @@
+module Translatomatic::TMX
+  class TranslationUnit
+
+    # @return [Array<Translatomatic::String>] Strings in this translation unit
+    attr_reader :strings
+
+    # @param [Array<Translatomatic::String>] list of strings
+    def initialize(strings)
+      @strings = strings || []
+    end
+
+    # Test translation unit validity.
+    # A translation unit must contain at least two strings.
+    # @return [boolean] true if this translation unit is valid
+    def valid?
+      @strings.length >= 2
+    end
+  end
+end

data/lib/translatomatic/tmx.rb

@@ -0,0 +1,4 @@
+module Translatomatic::TMX; end
+
+require 'translatomatic/tmx/translation_unit'
+require 'translatomatic/tmx/document'

data/lib/translatomatic/translation_result.rb

@@ -1,68 +1,86 @@
-require 'set'
-
-module Translatomatic
-  class TranslationResult
-
-    # Translation results
-    # @return [Hash<String,String>] Translation results
-    attr_reader :properties
-
-    # @return [Locale] The locale of the original strings
-    attr_reader :from_locale
-
-    # @return [Locale] The target locale
-    attr_reader :to_locale
-
-    # @return [Set<String>] Untranslated strings
-    attr_reader :untranslated
-
-    # Create a translation result
-    # @param [Hash<String,String>] properties Untranslated properties
-    # @param [Locale] from_locale The locale of the untranslated strings
-    # @param [Locale] to_locale The target locale
-    def initialize(properties, from_locale, to_locale)
-      @properties = properties.dup
-      @value_to_keys = {}
-      @untranslated = Set.new
-      properties.each do |key, value|
-        @untranslated << value
-        keylist = (@value_to_keys[value] ||= [])
-        keylist << key
-      end
+require 'set'
+
+module Translatomatic
+  class TranslationResult
+
+    # Translation results
+    # @return [Hash<String,String>] Translation results
+    attr_reader :properties
+
+    # @return [Locale] The locale of the original strings
+    attr_reader :from_locale
+
+    # @return [Locale] The target locale
+    attr_reader :to_locale
+
+    # @return [Set<String>] Untranslated strings
+    attr_reader :untranslated
+
+    # Create a translation result
+    # @param [Hash<String,String>] properties Untranslated properties
+    # @param [Locale] from_locale The locale of the untranslated strings
+    # @param [Locale] to_locale The target locale
+    def initialize(properties, from_locale, to_locale)
+      @value_to_keys = {}
+      @untranslated = Set.new
       @from_locale = from_locale
       @to_locale = to_locale
-    end
 
-    # Update result with a list of translated strings.
-    # @param [Array<String>] original Original strings
-    # @param [Array<String>] translated Translated strings
-    # @return [void]
-    def update_strings(original, translated)
+      # duplicate strings
+      @properties = properties.transform_values { |i| i.dup }
+
+      properties.each do |key, value|
+        # split property value into sentences
+        string = string(value, from_locale)
+        string.sentences.each do |sentence|
+          @untranslated << sentence
+          keylist = (@value_to_keys[sentence.to_s] ||= [])
+          keylist << key
+        end
+      end
+    end
+
+    # Update result with a list of translated strings.
+    # @param [Array<String>] original Original strings
+    # @param [Array<String>] translated Translated strings
+    # @return [void]
+    def update_strings(original, translated)
       raise "strings length mismatch" unless original.length == translated.length
-      original.zip(translated).each do |text1, text2|
-        update(text1, text2)
+
+      # create list of [from, to] text conversions
+      conversions = []
+      original.zip(translated).each do |text1, text2|
+        conversions << [text1, text2]
       end
-    end
 
-    # Update result with texts from the database.
-    # @param [Array<Translatomatic::Model::Text>] list Texts from database
-    # @return [void]
-    def update_db_strings(list)
-      list.each do |t|
-        original = t.from_text.value
-        translated = t.value
-        update(original, translated)
+      # sort conversion list by largest offset first so that we replace
+      # from the end of the string to the front, so substring offsets
+      # are correct in the target string.
+      conversions.sort_by! do |t1, t2|
+        t1.respond_to?(:offset) ? -t1.offset : 0
       end
-    end
 
+      conversions.each do |text1, text2|
+        update(text1, text2)
+      end
+    end
+
     private
 
-    def update(original, translated)
-      keys = @value_to_keys[original]
+    include Translatomatic::Util
+
+    def update(original, translated)
+      keys = @value_to_keys[original.to_s]
       raise "no key mapping for text '#{original}'" unless keys
-      keys.each { |key| @properties[key] = translated }
-
-      @untranslated.delete(original)
-    end
-  end
-end
+      keys.each do |key|
+        if original.kind_of?(Translatomatic::String) && original.substring?
+          @properties[key][original.offset, original.length] = translated
+        else
+          @properties[key] = translated
+        end
+      end
+
+      @untranslated.delete(original)
+    end
+  end
+end