tr8n_core 4.0.7 → 4.0.9

data/lib/tr8n/language.rb

@@ -36,7 +36,7 @@ class Tr8n::Language < Tr8n::Base
   has_many    :contexts, :cases
 
   def fetch
-    update_attributes(application.api_client.get("language", {:locale => locale}))
+    update_attributes(application.api_client.get("language", {:locale => locale}, {:cache_key => "#{locale}/language"}))
     self
   end
 
@@ -119,133 +119,46 @@ class Tr8n::Language < Tr8n::Base
 
     params = Tr8n::Utils.normalize_tr_params(label, description, tokens, options)
 
-    key_locale = hash_value(params[:options], :locale) || hash_value(Tr8n.session.block_options, :locale) || Tr8n.config.default_locale
-    key_level = hash_value(params[:options], :level) || hash_value(Tr8n.session.block_options, :level) || Tr8n.config.default_level
-
-    temp_key = Tr8n::TranslationKey.new({
-        :application  => application,
-        :label        => params[:label],
-        :description  => params[:description],
-        :locale       => key_locale,
-        :level        => key_level,
-        :translations => []
+    translation_key = Tr8n::TranslationKey.new({
+      :application  => application,
+      :label        => params[:label],
+      :description  => params[:description],
+      :locale       => hash_value(params[:options], :locale) || hash_value(Tr8n.session.block_options, :locale) || Tr8n.config.default_locale,
+      :level        => hash_value(params[:options], :level) || hash_value(Tr8n.session.block_options, :level) || Tr8n.config.default_level,
+      :translations => []
     })
 
     #Tr8n.logger.info("Translating " + params[:label] + " from: " + key_locale.inspect + " to " + locale.inspect)
 
     params[:tokens].merge!(:viewing_user => Tr8n.session.current_user)
 
-    if Tr8n.config.disabled? or self.locale == key_locale
-      return temp_key.substitute_tokens(params[:label], params[:tokens], self, params[:options]).tr8n_translated
-    end
-
-    translation_key = application.translation_key(temp_key.key)
-    if translation_key
-      return translation_key.translate(self, params[:tokens], params[:options])
-    end
-
-    # no cache or translator is using inline mode - use service, otherwise load from cache
-    if Tr8n.config.disabled? or (Tr8n.session.current_translator and Tr8n.session.current_translator.inline?)
-      return translate_from_service(temp_key, params[:tokens], params[:options]).tr8n_translated  
-    end
-
-    translate_from_cache(temp_key, params[:tokens], params[:options]).tr8n_translated
-  end
-  alias :tr :translate
-
-  def translate_from_cache(translation_key, tokens, options)
-    # In most scenarios translations should be cached by source
-    if Tr8n.cache.cached_by_source?
-      source_key = current_source(options)
-      cache_key = Tr8n::Source.cache_key(source_key, locale);
-      source = Tr8n.cache.fetch(cache_key)
-
-      # if it came from cache, it will be full of translation keys with translations for the locale
-      if source
-        translation_keys = source.translation_keys || {}
-      elsif Tr8n.cache.read_only?
-        translation_keys = {}
-      else
-        # get the source info from the application
-        source = application.source(source_key)
-        translation_keys = source.fetch_translations_for_language(self, options)
-        Tr8n.cache.store(cache_key, source)
-      end
-
-      if translation_keys[translation_key.key]
-        translation_key = translation_keys[translation_key.key]
-        return translation_key.translate(self, tokens, options)
-      end
-
-      translation_key.translations = {locale => []}
-      application.cache_translation_key(translation_key)
-      return translation_key.translate(self, tokens, options)
-    end
-
-    # CDB allows for caching by key
-    cache_key = Tr8n::TranslationKey.cache_key(translation_key.label, translation_key.description, locale)
-    translations = Tr8n.cache.fetch(cache_key)
-
-    if translations.nil?
-      if Tr8n.cache.read_only?
-        translation_key.translations = {locale => []}
-        application.cache_translation_key(translation_key)
-        return translation_key.translate(self, tokens, options)
-      end
-
-      translation_key = translation_key.fetch_translations(self, options)
-      Tr8n.cache.store(cache_key, translation_key.translations(self))
-      return translation_key.translate(self, tokens, options)
+    if Tr8n.config.disabled? or self.locale == translation_key.locale
+      return translation_key.substitute_tokens(
+        params[:label],
+        params[:tokens],
+        self,
+        params[:options]
+      ).tr8n_translated
     end
 
-    unless translations.is_a?(Array)
-      translations = [translations]
-    end
-
-    translation_key.translations = {locale => translations}
-    application.cache_translation_key(translation_key)
-    translation_key.translate(self, tokens, options)
-  end
+    # should this be done only for testing?
+    cached_key = application.translation_key(translation_key.key)
+    return cached_key.translate(self, params[:tokens], params[:options]) if cached_key
 
-  def translate_from_service(translation_key, tokens, options)
     source_key = current_source(options)
+    source = application.source(source_key, locale)
 
-    source = application.source(source_key)
-    source_translation_keys = source.fetch_translations_for_language(self, options)
-    if source_translation_keys[translation_key.key]
-      translation_key = source_translation_keys[translation_key.key]
+    cached_key = source ? source.translation_key(translation_key.key) : nil
+    if cached_key
+      translation_key = cached_key
     else
-      application.register_missing_key(translation_key, source)
+      application.register_missing_key(source_key, translation_key)
       memory_cached_translation_key = application.translation_key(translation_key.key)
       translation_key = memory_cached_translation_key if memory_cached_translation_key
     end
 
-    translation_key.translate(self, tokens, options)
-  end
-
-  #######################################################################################################
-  ##  Cache Methods
-  #######################################################################################################
-
-  def self.cache_prefix
-    'l@'
-  end
-
-  def self.cache_key(locale)
-    "#{cache_prefix}_[#{locale}]"
-  end
-
-  def to_cache_hash
-    hash = to_hash(:locale, :name, :english_name, :native_name, :right_to_left, :flag_url)
-    hash[:contexts] = {}
-    contexts.each do |name, value|
-      hash[:contexts][name] = value.to_cache_hash
-    end
-    hash[:cases] = {}
-    cases.each do |name, value|
-      hash[:cases][name] = value.to_cache_hash
-    end
-    hash
+    translation_key.translate(self, params[:tokens], params[:options]).tr8n_translated
   end
+  alias :tr :translate
 
 end

data/lib/tr8n/language_case.rb

@@ -64,7 +64,7 @@ class Tr8n::LanguageCase < Tr8n::Base
     html_tokens = value.scan(TR8N_HTML_TAGS_REGEX).uniq
     sanitized_value = value.gsub(TR8N_HTML_TAGS_REGEX, "")
 
-    if application.to_sym == :phrase
+    if application.to_s == "phrase"
       words = [sanitized_value]
     else
       words = sanitized_value.split(/[\s\/\\]/).uniq

data/lib/tr8n/session.rb

@@ -47,15 +47,8 @@ module Tr8n
       host    ||= Tr8n.config.application[:host]
 
       Tr8n.cache.reset_version
-
-      self.application = Tr8n.cache.fetch(Tr8n::Application.cache_key(key)) do
-        Tr8n.logger.info("Initializing application...")
-        Tr8n::Application.new(:host => host, :key => key, :secret => secret).fetch
-      end
-
-      self.current_source = "/tr8n/core"
-
-      self.application
+      self.current_source = "/tr8n/core" # default source
+      self.application = Tr8n::Application.new(:host => host, :key => key, :secret => secret).fetch
     end
 
     def reset

data/lib/tr8n/source.rb

@@ -36,25 +36,28 @@ class Tr8n::Source < Tr8n::Base
   has_many    :translation_keys
 
   def self.normalize(url)
-    return nil if url.nil? or url == ""
+    return nil if url.nil? or url == ''
     uri = URI.parse(url)
     path = uri.path
-    return "/" if uri.path.nil? or uri.path == ""
-    return path if path == "/"
+    return '/' if uri.path.nil? or uri.path == ''
+    return path if path == '/'
 
     # always must start with /
-    path = "/#{path}" if path[0] != "/"
+    path = "/#{path}" if path[0] != '/'
     # should not end with /
-    path = path[0..-2] if path[-1] == "/"
+    path = path[0..-2] if path[-1] == '/'
     path
   end
 
+  def self.cache_key(key, locale)
+    "#{locale}/[#{key.gsub(/[\.\/]/, '-')}]"
+  end
+
   def initialize(attrs = {})
     super
 
-    self.translation_keys = nil
+    self.translation_keys = {}
     if hash_value(attrs, :translation_keys)
-      self.translation_keys = {}
       hash_value(attrs, :translation_keys).each do |tk|
         tkey = Tr8n::TranslationKey.new(tk.merge(:application => application))
         self.translation_keys[tkey.key] = application.cache_translation_key(tkey)
@@ -62,43 +65,8 @@ class Tr8n::Source < Tr8n::Base
     end
   end
 
-  def fetch_translations_for_language(language, options = {})
-    return translation_keys if translation_keys
-
-    keys_with_translations = application.api_client.get("source/translations",
-                                             {:source => source, :locale => language.locale},
-                                             {:class => Tr8n::TranslationKey, :attributes => {:application => application}})
-
-    self.attributes[:translation_keys] = {}
-
-    keys_with_translations.each do |tkey|
-      self.attributes[:translation_keys][tkey.key] = application.cache_translation_key(tkey)
-    end
-
-    self.attributes[:translation_keys]
-  end
-
-  #######################################################################################################
-  ##  Cache Methods
-  #######################################################################################################
-
-  def self.cache_prefix
-    's@'
-  end
-
-  def self.cache_key(source_key, locale)
-    "#{cache_prefix}_[#{locale}]_[#{source_key}]"
-  end
-
-  def to_cache_hash
-    hash = to_hash(:source, :url, :name, :description)
-    if translation_keys and translation_keys.any?
-      hash[:translation_keys] = []
-      translation_keys.values.each do |tkey|
-        hash[:translation_keys] << tkey.to_cache_hash
-      end
-    end
-    hash
+  def translation_key(key)
+    self.translation_keys[key]
   end
 
 end

data/lib/tr8n/{tokens/data_tokenizer.rb → tokenizers/data.rb}

@@ -46,14 +46,15 @@
 # [link: {user.name}]
 #
 #######################################################################
+
 module Tr8n
-  module Tokens
-    class DataTokenizer
+  module Tokenizers
+    class Data
 
       attr_accessor :text, :context, :tokens, :opts
 
       def self.supported_tokens
-        [Tr8n::Tokens::Data, Tr8n::Tokens::Hidden, Tr8n::Tokens::Method, Tr8n::Tokens::Transform]
+        [Tr8n::Tokens::Data, Tr8n::Tokens::Method, Tr8n::Tokens::Transform]
       end
 
       def self.required?(label)

data/lib/tr8n/{tokens/decoration_tokenizer.rb → tokenizers/decoration.rb}

@@ -46,9 +46,10 @@
 # [link: {user.name}]
 #
 #######################################################################
+
 module Tr8n
-  module Tokens
-    class DecorationTokenizer
+  module Tokenizers
+    class Decoration
 
       attr_reader :tokens, :fragments, :context, :text, :opts
 
@@ -61,7 +62,7 @@ module Tr8n
       RE_TEXT              = '[^\[\]]+' #'[\w\s!.:{}\(\)\|,?]*'
 
       def self.required?(label)
-        label.index("[")
+        label.index('[')
       end
 
       def initialize(text, context = {}, opts = {})
@@ -131,7 +132,7 @@ module Tr8n
         default_decoration = default_decoration.clone
         decoration_token_values = context[token_name.to_sym] || context[token_name.to_s]
 
-        default_decoration.gsub!("{$0}", token_value.to_s)
+        default_decoration.gsub!('{$0}', token_value.to_s)
 
         if decoration_token_values.is_a?(Hash)
           decoration_token_values.keys.each do |key|
@@ -163,7 +164,7 @@ module Tr8n
           end
 
           if method.is_a?(String)
-            return method.to_s.gsub("{$0}", value)
+            return method.to_s.gsub('{$0}', value)
           end
 
           Tr8n.logger.error("Invalid decoration token value for #{token} in #{text}")

data/lib/tr8n/tokenizers/dom.rb

@@ -0,0 +1,346 @@
+# encoding: UTF-8
+#--
+# Copyright (c) 2014 Michael Berkovich, TranslationExchange.com
+#
+#  _______                  _       _   _             ______          _
+# |__   __|                | |     | | (_)           |  ____|        | |
+#    | |_ __ __ _ _ __  ___| | __ _| |_ _  ___  _ __ | |__  __  _____| |__   __ _ _ __   __ _  ___
+#    | | '__/ _` | '_ \/ __| |/ _` | __| |/ _ \| '_ \|  __| \ \/ / __| '_ \ / _` | '_ \ / _` |/ _ \
+#    | | | | (_| | | | \__ \ | (_| | |_| | (_) | | | | |____ >  < (__| | | | (_| | | | | (_| |  __/
+#    |_|_|  \__,_|_| |_|___/_|\__,_|\__|_|\___/|_| |_|______/_/\_\___|_| |_|\__,_|_| |_|\__, |\___|
+#                                                                                        __/ |
+#                                                                                       |___/
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'nokogiri'
+
+module Tr8n
+  module Tokenizers
+    class Dom
+
+      HTML_SPECIAL_CHAR_REGEX = /(&[^;]*;)/
+      INDEPENDENT_NUMBER_REGEX = /^(\d+)$|^(\d+[.,;\s])|(\s\d+)$|(\s\d+[,;\s])/
+      VERBOSE_DATE_REGEX = /(((Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)|(January|February|March|April|May|June|July|August|September|October|November|December))\\s\\d+(,\\s\\d+)*(,*\\sat\\s\\d+:\\d+(\\sUTC))*)/
+
+      attr_accessor :context, :tokens, :options
+
+      def initialize(context = {}, options = {})
+        self.context = context
+        self.options = options
+        reset_context
+      end
+
+      def translate(doc)
+        translate_tree(doc.is_a?(String) ? Nokogiri::HTML.fragment(doc) : doc)
+      end
+
+      def translate_tree(node)
+        if non_translatable_node?(node)
+          return node.children.first.inner_text if node.children.count == 1
+          return ''
+        end
+
+        return translate_tml(node.inner_text) if node.type == 3
+
+        html = ''
+        buffer = ''
+
+        node.children.each do |child|
+          if child.type == 3
+            buffer += child.inner_text
+          elsif inline_node?(child) and has_inline_or_text_siblings?(child) and !between_separators?(child)
+            buffer += generate_tml_tags(child)
+          elsif separator_node?(child)
+            html += translate_tml(buffer) if buffer != ''
+            html += generate_html_token(child)
+            buffer = ''
+          else
+            html += translate_tml(buffer) if buffer != ''
+
+            container_value = translate_tree(child)
+            if ignored_node?(child)
+              html += container_value
+            else
+              html += generate_html_token(child, container_value)
+            end
+
+            buffer = ''
+          end
+        end
+
+        html += translate_tml(buffer) if buffer != ''
+        html
+      end
+
+      def non_translatable_node?(node)
+        return false unless node
+        return true if node.type == 1 && (option('nodes.scripts') || []).index(node.name.downcase)
+        return true if node.type == 1 && node.children.length === 0 && node.inner_text == ''
+        false
+      end
+
+      def translate_tml(tml)
+        return tml if empty_string?(tml)
+        tml = generate_data_tokens(tml)
+
+        if option('split_sentences')
+          sentences = Tr8n::Utils.split_sentences(tml)
+          translation = tml
+          sentences.each do |sentence|
+            sentence_translation = option('debug') ? debug_translation(sentence) : Tr8n.session.current_language.translate(sentence, tokens, options)
+            translation = translation.gsub(sentence, sentence_translation)
+          end
+          reset_context
+          return translation
+        end
+
+        tml = tml.gsub(/[\n]/, '').gsub(/\s\s+/, ' ').strip
+
+        translation = option('debug') ? debug_translation(tml) : Tr8n.session.current_language.translate(tml, tokens, options)
+        reset_context
+        translation
+      end
+
+      def has_child_nodes?(node)
+        node.children and node.children.length > 0
+      end
+
+      def between_separators?(node)
+        (separator_node?(node.previous_sibling) and !valid_text_node?(node.next_sibling)) or
+        (separator_node?(node.next_sibling) and !valid_text_node?(node.previous_sibling))
+      end
+
+      def generate_tml_tags(node)
+        buffer = ''
+        node.children.each do |child|
+          if child.type == 3
+            buffer += child.inner_text
+          else
+            buffer += generate_tml_tags(child)
+          end
+        end
+
+        token_context = generate_html_token(node)
+        token = contextualize(adjust_name(node), token_context)
+        value = sanitize_value(buffer)
+
+        return '{' + token + '}' if self_closing_node?(node)
+        return '[' + token + ': ' + value + ']' if short_token?(token, value)
+
+        '[' + token + ']' + value + '[/' + token + ']'
+      end
+
+      def option(name)
+        value = Tr8n::Utils.hash_value(self.options, name)
+        value || Tr8n.config.translator_option(name)
+      end
+
+      def debug_translation(translation)
+        option('debug_format').gsub('{$0}', translation)
+      end
+
+      def empty_string?(tml)
+        tml = tml.gsub(/[\s\n\r\t]/, '')
+        tml == ''
+      end
+
+      def reset_context
+        self.tokens = {}.merge(self.context)
+      end
+
+      def short_token?(token, value)
+        option('nodes.short').index(token.downcase) || value.length < 20
+      end
+
+      def only_child?(node)
+        return false unless node.parent
+        node.parent.children.count == 1
+      end
+
+      def has_inline_or_text_siblings?(node)
+        return false unless node.parent
+
+        node.parent.children.each do |child|
+          unless child == node
+            return true if inline_node?(child) || valid_text_node?(child)
+          end
+        end
+
+        false
+      end
+
+      def inline_node?(node)
+        (
+          node.type == 1 and
+          (option('nodes.inline') || []).index(node.name.downcase) and
+          !only_child?(node)
+        )
+      end
+
+      def container_node?(node)
+        node.type == 1 && !inline_node?(node)
+      end
+
+      def self_closing_node?(node)
+        !node.children || !node.children.first
+      end
+    
+      def ignored_node?(node)
+        return true if (node.type != 1)
+        (option('nodes.ignored') || []).index(node.name.downcase)
+      end
+
+      def valid_text_node?(node)
+        return false unless node
+        node.type == 3 && !empty_string?(node.inner_text)
+      end
+
+      def separator_node?(node)
+        return false unless node
+        node.type == 1 && (option('nodes.splitters') || []).index(node.name.downcase)
+      end
+
+      def sanitize_value(value)
+        value.gsub(/^\s+/, '')
+      end
+
+      def replace_special_characters(text)
+        return text if option('data_tokens.special')
+
+        matches = text.match(HTML_SPECIAL_CHAR_REGEX)
+        matches.each do  |match|
+          token = match[1, - 2]
+          self.context[token] = match
+          text = text.gsub(match, "{#{token}}")
+        end
+
+        text
+      end
+
+      def generate_data_tokens(text)
+        return text unless option('data_tokens.numeric')
+
+        matches = text.match(INDEPENDENT_NUMBER_REGEX) || []
+        token_name = option('data_tokens.numeric_name')
+
+        matches.each do |match|
+          value = match.gsub(/[.,;\s]/, '')
+          token = contextualize(token_name, value.to_i)
+          replacement = match.replace(value, "{#{token}}")
+          text = text.gsub(match, match.gsub(value, replacement))
+        end
+
+        text
+      end
+
+      def generate_html_token(node, value = nil)
+        name = node.name.downcase
+        attributes = node.attributes
+        attributes_hash = {}
+        value = (!value ? '{$0}' : value)
+
+        if attributes.length == 0
+          if self_closing_node?(node)
+            return '<' + name + '/>' if %w(br hr).index(name)
+            return '<' + name + '>' + '</' + name + '>'
+          end
+          return '<' + name + '>' + value + '</' + name + '>'
+        end
+
+        attributes.each do |name, attribute|
+          attributes_hash[name] = attribute.value
+        end
+
+        keys = attributes_hash.keys.sort
+
+        attr = []
+        keys.each do |key|
+          quote = attributes_hash[key].index("'") ? '"' : "'"
+          attr << (key + '=' + quote + attributes_hash[key] + quote)
+        end
+        attr = attr.join(' ')
+
+        return '<' + name + ' ' + attr + '>' + '</' + name + '>' if self_closing_node?(node)
+        '<' + name + ' ' + attr + '>' + value + '</' + name + '>'
+      end
+
+      def adjust_name(node)
+        name = node.name.downcase
+        map = option('name_mapping')
+        map[name.to_sym] ? map[name.to_sym] : name
+      end
+
+      def contextualize(name, context)
+        if self.tokens[name] and self.tokens[name] != context
+          index = 0
+          matches = name.match(/\d+$/)
+          if matches and matches.length > 0
+            index = matches[matches.length-1].to_i
+            name = name.gsub('' + index, '')
+          end
+          name += (index + 1).to_s
+          return contextualize(name, context)
+        end
+
+        self.tokens[name] = context
+        name
+      end
+
+      def debug(doc)
+        self.doc = doc
+        debug_tree(self.doc, 0)
+      end
+
+      def debug_tree(node, depth)
+        padding = ('=' * (depth+1))
+
+        Tr8n.logger.log(padding + '=> ' + (node) + ': ' + node_info(node))
+
+        (node.children || []).each do |child|
+          debug_tree(child, depth+1)
+        end
+      end
+
+      def node_info(node)
+        info = []
+        info << node.type
+
+        info << node.tagName if node.type == 1
+
+        if inline_node?(node)
+          info << 'inline'
+          if has_inline_or_text_siblings?(node)
+            info << 'sentence'
+          else
+            info << 'only translatable'
+          end
+        end
+
+        info << 'self closing' if self_closing_node?(node)
+        info << 'only child' if only_child?(node)
+
+        return "[#{info.join(', ')}]: " + node.inner_text if node.type == 3
+        "[#{info.join(', ')}]"
+      end
+
+    end
+  end
+end