tml 4.3.1

data/lib/tml/source.rb

@@ -0,0 +1,106 @@
+# encoding: UTF-8
+#--
+# Copyright (c) 2015 Translation Exchange, Inc
+#
+#  _______                  _       _   _             ______          _
+# |__   __|                | |     | | (_)           |  ____|        | |
+#    | |_ __ __ _ _ __  ___| | __ _| |_ _  ___  _ __ | |__  __  _____| |__   __ _ _ __   __ _  ___
+#    | | '__/ _` | '_ \/ __| |/ _` | __| |/ _ \| '_ \|  __| \ \/ / __| '_ \ / _` | '_ \ / _` |/ _ \
+#    | | | | (_| | | | \__ \ | (_| | |_| | (_) | | | | |____ >  < (__| | | | (_| | | | | (_| |  __/
+#    |_|_|  \__,_|_| |_|___/_|\__,_|\__|_|\___/|_| |_|______/_/\_\___|_| |_|\__,_|_| |_|\__, |\___|
+#                                                                                        __/ |
+#                                                                                       |___/
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'digest/md5'
+
+class Tml::Source < Tml::Base
+  belongs_to  :application
+  attributes  :key, :source, :url, :name, :description
+  has_many    :translations
+
+  def self.normalize(url)
+    return nil if url.nil? or url == ''
+    uri = URI.parse(url)
+    path = uri.path
+    return '/' if uri.path.nil? or uri.path == ''
+    return path if path == '/'
+
+    # always must start with /
+    path = "/#{path}" if path[0] != '/'
+    # should not end with /
+    path = path[0..-2] if path[-1] == '/'
+    path
+  end
+
+  def self.generate_key(source)
+    "#{Digest::MD5.hexdigest("#{source}")}~"[0..-2]
+  end
+
+  def self.cache_key(locale, source)
+    File.join(locale, 'sources', source.split('/'))
+  end
+
+  def initialize(attrs = {})
+    super
+    self.key ||= Tml::Source.generate_key(attrs[:source])
+  end
+
+  def fetch_translations(locale)
+    self.translations ||= {}
+    return self if self.translations[locale]
+
+    self.translations[locale] = {}
+
+    results = self.application.api_client.get(
+      "sources/#{self.key}/translations",
+      {:locale => locale, :per_page => 10000},
+      {:cache_key => Tml::Source.cache_key(locale, self.source)}
+    )
+
+    results.each do |key, data|
+      translations_data = data.is_a?(Hash) ? data['translations'] : data
+      self.translations[locale][key] = translations_data.collect do |t|
+        Tml::Translation.new(
+          :locale => t['locale'] || locale,
+          :label => t['label'],
+          :context => t['context']
+        )
+      end
+    end
+
+    self
+  rescue Tml::Exception => ex
+    self
+  end
+
+  def cached_translations(locale, key)
+    self.translations ||= {}
+    self.translations[locale] ||= {}
+    self.translations[locale][key]
+  end
+
+  def reset_cache
+    application.languages.each do |lang|
+      Tml.cache.delete(Tml::Source.cache_key(lang.locale, self.source))
+    end
+  end
+end

data/lib/tml/tokenizers/data.rb

@@ -0,0 +1,96 @@
+# encoding: UTF-8
+#--
+# Copyright (c) 2015 Translation Exchange, Inc
+#
+#  _______                  _       _   _             ______          _
+# |__   __|                | |     | | (_)           |  ____|        | |
+#    | |_ __ __ _ _ __  ___| | __ _| |_ _  ___  _ __ | |__  __  _____| |__   __ _ _ __   __ _  ___
+#    | | '__/ _` | '_ \/ __| |/ _` | __| |/ _ \| '_ \|  __| \ \/ / __| '_ \ / _` | '_ \ / _` |/ _ \
+#    | | | | (_| | | | \__ \ | (_| | |_| | (_) | | | | |____ >  < (__| | | | (_| | | | | (_| |  __/
+#    |_|_|  \__,_|_| |_|___/_|\__,_|\__|_|\___/|_| |_|______/_/\_\___|_| |_|\__,_|_| |_|\__, |\___|
+#                                                                                        __/ |
+#                                                                                       |___/
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+#######################################################################
+#
+# Decoration Token Forms:
+#
+# [link: click here]
+# or
+# [link] click here [/link]
+#
+# Decoration Tokens Allow Nesting:
+#
+# [link: {count} {_messages}]
+# [link: {count||message}]
+# [link: {count||person, people}]
+# [link: {user.name}]
+#
+#######################################################################
+
+module Tml
+  module Tokenizers
+    class Data
+
+      attr_accessor :text, :context, :tokens, :opts
+
+      def self.supported_tokens
+        [Tml::Tokens::Data, Tml::Tokens::Method, Tml::Tokens::Transform]
+      end
+
+      def self.required?(label)
+        label.index("{")
+      end
+
+      def initialize(text, context={}, opts={})
+        self.text = text
+        self.context = context
+        self.opts = opts
+        self.tokens = []
+        tokenize
+      end
+
+      def tokenize
+        self.tokens = []
+        self.class.supported_tokens.each do |klass|
+          self.tokens << klass.parse(self.text)
+        end
+        self.tokens.flatten!.uniq!
+      end
+
+      def token_allowed?(token)
+        return true unless opts[:allowed_tokens]
+        not opts[:allowed_tokens][token.name].nil?
+      end
+
+      def substitute(language, options = {})
+        label = self.text
+        tokens.each do |token|
+          next unless token_allowed?(token)
+          label = token.substitute(label, context, language, options)
+        end
+        label
+      end
+
+    end
+  end
+end

data/lib/tml/tokenizers/decoration.rb

@@ -0,0 +1,204 @@
+# encoding: UTF-8
+#--
+# Copyright (c) 2015 Translation Exchange, Inc
+#
+#  _______                  _       _   _             ______          _
+# |__   __|                | |     | | (_)           |  ____|        | |
+#    | |_ __ __ _ _ __  ___| | __ _| |_ _  ___  _ __ | |__  __  _____| |__   __ _ _ __   __ _  ___
+#    | | '__/ _` | '_ \/ __| |/ _` | __| |/ _ \| '_ \|  __| \ \/ / __| '_ \ / _` | '_ \ / _` |/ _ \
+#    | | | | (_| | | | \__ \ | (_| | |_| | (_) | | | | |____ >  < (__| | | | (_| | | | | (_| |  __/
+#    |_|_|  \__,_|_| |_|___/_|\__,_|\__|_|\___/|_| |_|______/_/\_\___|_| |_|\__,_|_| |_|\__, |\___|
+#                                                                                        __/ |
+#                                                                                       |___/
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+#######################################################################
+#
+# Decoration Token Forms:
+#
+# [link: click here]
+# or
+# [link] click here [/link]
+#
+# Decoration Tokens Allow Nesting:
+#
+# [link: {count} {_messages}]
+# [link: {count||message}]
+# [link: {count||person, people}]
+# [link: {user.name}]
+#
+#######################################################################
+
+module Tml
+  module Tokenizers
+    class Decoration
+
+      attr_reader :tokens, :fragments, :context, :text, :opts
+
+      RESERVED_TOKEN       = 'tml'
+
+      RE_SHORT_TOKEN_START = '\[[\w]*:'
+      RE_SHORT_TOKEN_END   = '\]'
+      RE_LONG_TOKEN_START  = '\[[\w]*\]'
+      RE_LONG_TOKEN_END    = '\[\/[\w]*\]'
+      RE_TEXT              = '[^\[\]]+' #'[\w\s!.:{}\(\)\|,?]*'
+
+      def self.required?(label)
+        label.index('[')
+      end
+
+      def initialize(text, context = {}, opts = {})
+        @text = "[#{RESERVED_TOKEN}]#{text}[/#{RESERVED_TOKEN}]"
+        @context = context
+        @opts = opts
+        tokenize
+      end
+
+      def tokenize
+        re = [RE_SHORT_TOKEN_START,
+              RE_SHORT_TOKEN_END,
+              RE_LONG_TOKEN_START,
+              RE_LONG_TOKEN_END,
+              RE_TEXT].join('|')
+        @fragments = text.scan(/#{re}/)
+        @tokens = []
+      end
+
+      def parse
+        return @text unless fragments
+        token = fragments.shift
+
+        if token.match(/#{RE_SHORT_TOKEN_START}/)
+          return parse_tree(token.gsub(/[\[:]/, ''), :short)
+        end
+
+        if token.match(/#{RE_LONG_TOKEN_START}/)
+          return parse_tree(token.gsub(/[\[\]]/, ''), :long)
+        end
+
+        token.to_s
+      end
+
+      def parse_tree(name, type = :short)
+        tree = [name]
+        @tokens << name unless (@tokens.include?(name) or name == RESERVED_TOKEN)
+
+        if type == :short
+          first = true
+          until fragments.first.nil? or fragments.first.match(/#{RE_SHORT_TOKEN_END}/)
+            value = parse
+            if first and value.is_a?(String)
+              value = value.lstrip
+              first = false
+            end
+            tree << value
+          end
+        elsif type == :long
+          until fragments.first.nil? or fragments.first.match(/#{RE_LONG_TOKEN_END}/)
+            tree << parse
+          end
+        end
+
+        fragments.shift
+        tree
+      end
+
+      def default_decoration(token_name, token_value)
+        default_decoration = Tml.config.default_token_value(normalize_token(token_name), :decoration)
+
+        unless default_decoration
+          Tml.logger.error("Invalid decoration token value for #{token_name} in #{text}")
+          return token_value
+        end
+
+        default_decoration = default_decoration.clone
+        decoration_token_values = context[token_name.to_sym] || context[token_name.to_s]
+
+        default_decoration.gsub!('{$0}', token_value.to_s)
+
+        if decoration_token_values.is_a?(Hash)
+          decoration_token_values.keys.each do |key|
+            default_decoration.gsub!("{$#{key}}", decoration_token_values[key].to_s)
+          end
+        end
+
+        default_decoration
+      end
+
+      def allowed_token?(token)
+        return true if opts[:allowed_tokens].nil?
+        opts[:allowed_tokens].include?(token)
+      end
+
+      def apply(token, value)
+        return value if token == RESERVED_TOKEN
+        return value unless allowed_token?(token)
+
+        method = context[token.to_sym] || context[token.to_s]
+
+        if method
+          if method.is_a?(Proc)
+            return method.call(value)
+          end
+
+          if method.is_a?(Array) or method.is_a?(Hash)
+            return default_decoration(token, value)
+          end
+
+          if method.is_a?(String)
+            return method.to_s.gsub('{$0}', value)
+          end
+
+          Tml.logger.error("Invalid decoration token value for #{token} in #{text}")
+          return value
+        end
+
+        if Tml.config.default_token_value(normalize_token(token), :decoration)
+          return default_decoration(token, value)
+        end
+
+        Tml.logger.error("Missing decoration token value for #{token} in #{text}")
+        value
+      end
+
+      def normalize_token(name)
+        name.to_s.gsub(/(\d)*$/, '')
+      end
+
+      def evaluate(expr)
+        unless expr.is_a?(Array)
+          return expr
+        end
+
+        token = expr[0]
+        args = expr.drop(1)
+        value = args.map { |a| self.evaluate(a) }.join('')
+
+        apply(token, value)
+      end
+
+      def substitute
+        evaluate(parse)
+      end
+
+    end
+  end
+end

data/lib/tml/tokenizers/dom.rb

@@ -0,0 +1,346 @@
+# encoding: UTF-8
+#--
+# Copyright (c) 2015 Translation Exchange, Inc
+#
+#  _______                  _       _   _             ______          _
+# |__   __|                | |     | | (_)           |  ____|        | |
+#    | |_ __ __ _ _ __  ___| | __ _| |_ _  ___  _ __ | |__  __  _____| |__   __ _ _ __   __ _  ___
+#    | | '__/ _` | '_ \/ __| |/ _` | __| |/ _ \| '_ \|  __| \ \/ / __| '_ \ / _` | '_ \ / _` |/ _ \
+#    | | | | (_| | | | \__ \ | (_| | |_| | (_) | | | | |____ >  < (__| | | | (_| | | | | (_| |  __/
+#    |_|_|  \__,_|_| |_|___/_|\__,_|\__|_|\___/|_| |_|______/_/\_\___|_| |_|\__,_|_| |_|\__, |\___|
+#                                                                                        __/ |
+#                                                                                       |___/
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'nokogiri'
+
+module Tml
+  module Tokenizers
+    class Dom
+
+      HTML_SPECIAL_CHAR_REGEX = /(&[^;]*;)/
+      INDEPENDENT_NUMBER_REGEX = /^(\d+)$|^(\d+[.,;\s])|(\s\d+)$|(\s\d+[,;\s])/
+      VERBOSE_DATE_REGEX = /(((Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)|(January|February|March|April|May|June|July|August|September|October|November|December))\\s\\d+(,\\s\\d+)*(,*\\sat\\s\\d+:\\d+(\\sUTC))*)/
+
+      attr_accessor :context, :tokens, :options
+
+      def initialize(context = {}, options = {})
+        self.context = context
+        self.options = options
+        reset_context
+      end
+
+      def translate(doc)
+        translate_tree(doc.is_a?(String) ? Nokogiri::HTML.fragment(doc) : doc)
+      end
+
+      def translate_tree(node)
+        if non_translatable_node?(node)
+          return node.children.first.inner_text if node.children.count == 1
+          return ''
+        end
+
+        return translate_tml(node.inner_text) if node.type == 3
+
+        html = ''
+        buffer = ''
+
+        node.children.each do |child|
+          if child.type == 3
+            buffer += child.inner_text
+          elsif inline_node?(child) and has_inline_or_text_siblings?(child) and !between_separators?(child)
+            buffer += generate_tml_tags(child)
+          elsif separator_node?(child)
+            html += translate_tml(buffer) if buffer != ''
+            html += generate_html_token(child)
+            buffer = ''
+          else
+            html += translate_tml(buffer) if buffer != ''
+
+            container_value = translate_tree(child)
+            if ignored_node?(child)
+              html += container_value
+            else
+              html += generate_html_token(child, container_value)
+            end
+
+            buffer = ''
+          end
+        end
+
+        html += translate_tml(buffer) if buffer != ''
+        html
+      end
+
+      def non_translatable_node?(node)
+        return false unless node
+        return true if node.type == 1 && (option('nodes.scripts') || []).index(node.name.downcase)
+        return true if node.type == 1 && node.children.length === 0 && node.inner_text == ''
+        false
+      end
+
+      def translate_tml(tml)
+        return tml if empty_string?(tml)
+        tml = generate_data_tokens(tml)
+
+        if option('split_sentences')
+          sentences = Tml::Utils.split_sentences(tml)
+          translation = tml
+          sentences.each do |sentence|
+            sentence_translation = option('debug') ? debug_translation(sentence) : Tml.session.current_language.translate(sentence, tokens, options)
+            translation = translation.gsub(sentence, sentence_translation)
+          end
+          reset_context
+          return translation
+        end
+
+        tml = tml.gsub(/[\n]/, '').gsub(/\s\s+/, ' ').strip
+
+        translation = option('debug') ? debug_translation(tml) : Tml.session.target_language.translate(tml, tokens, options)
+        reset_context
+        translation
+      end
+
+      def has_child_nodes?(node)
+        node.children and node.children.length > 0
+      end
+
+      def between_separators?(node)
+        (separator_node?(node.previous_sibling) and !valid_text_node?(node.next_sibling)) or
+        (separator_node?(node.next_sibling) and !valid_text_node?(node.previous_sibling))
+      end
+
+      def generate_tml_tags(node)
+        buffer = ''
+        node.children.each do |child|
+          if child.type == 3
+            buffer += child.inner_text
+          else
+            buffer += generate_tml_tags(child)
+          end
+        end
+
+        token_context = generate_html_token(node)
+        token = contextualize(adjust_name(node), token_context)
+        value = sanitize_value(buffer)
+
+        return '{' + token + '}' if self_closing_node?(node)
+        return '[' + token + ': ' + value + ']' if short_token?(token, value)
+
+        '[' + token + ']' + value + '[/' + token + ']'
+      end
+
+      def option(name)
+        value = Tml::Utils.hash_value(self.options, name)
+        value || Tml.config.translator_option(name)
+      end
+
+      def debug_translation(translation)
+        option('debug_format').gsub('{$0}', translation)
+      end
+
+      def empty_string?(tml)
+        tml = tml.gsub(/[\s\n\r\t]/, '')
+        tml == ''
+      end
+
+      def reset_context
+        self.tokens = {}.merge(self.context)
+      end
+
+      def short_token?(token, value)
+        option('nodes.short').index(token.downcase) || value.length < 20
+      end
+
+      def only_child?(node)
+        return false unless node.parent
+        node.parent.children.count == 1
+      end
+
+      def has_inline_or_text_siblings?(node)
+        return false unless node.parent
+
+        node.parent.children.each do |child|
+          unless child == node
+            return true if inline_node?(child) || valid_text_node?(child)
+          end
+        end
+
+        false
+      end
+
+      def inline_node?(node)
+        (
+          node.type == 1 and
+          (option('nodes.inline') || []).index(node.name.downcase) and
+          !only_child?(node)
+        )
+      end
+
+      def container_node?(node)
+        node.type == 1 && !inline_node?(node)
+      end
+
+      def self_closing_node?(node)
+        !node.children || !node.children.first
+      end
+    
+      def ignored_node?(node)
+        return true if (node.type != 1)
+        (option('nodes.ignored') || []).index(node.name.downcase)
+      end
+
+      def valid_text_node?(node)
+        return false unless node
+        node.type == 3 && !empty_string?(node.inner_text)
+      end
+
+      def separator_node?(node)
+        return false unless node
+        node.type == 1 && (option('nodes.splitters') || []).index(node.name.downcase)
+      end
+
+      def sanitize_value(value)
+        value.gsub(/^\s+/, '')
+      end
+
+      def replace_special_characters(text)
+        return text if option('data_tokens.special')
+
+        matches = text.match(HTML_SPECIAL_CHAR_REGEX)
+        matches.each do  |match|
+          token = match[1, - 2]
+          self.context[token] = match
+          text = text.gsub(match, "{#{token}}")
+        end
+
+        text
+      end
+
+      def generate_data_tokens(text)
+        return text unless option('data_tokens.numeric')
+
+        matches = text.match(INDEPENDENT_NUMBER_REGEX) || []
+        token_name = option('data_tokens.numeric_name')
+
+        matches.each do |match|
+          value = match.gsub(/[.,;\s]/, '')
+          token = contextualize(token_name, value.to_i)
+          replacement = match.replace(value, "{#{token}}")
+          text = text.gsub(match, match.gsub(value, replacement))
+        end
+
+        text
+      end
+
+      def generate_html_token(node, value = nil)
+        name = node.name.downcase
+        attributes = node.attributes
+        attributes_hash = {}
+        value = (!value ? '{$0}' : value)
+
+        if attributes.length == 0
+          if self_closing_node?(node)
+            return '<' + name + '/>' if %w(br hr).index(name)
+            return '<' + name + '>' + '</' + name + '>'
+          end
+          return '<' + name + '>' + value + '</' + name + '>'
+        end
+
+        attributes.each do |name, attribute|
+          attributes_hash[name] = attribute.value
+        end
+
+        keys = attributes_hash.keys.sort
+
+        attr = []
+        keys.each do |key|
+          quote = attributes_hash[key].index("'") ? '"' : "'"
+          attr << (key + '=' + quote + attributes_hash[key] + quote)
+        end
+        attr = attr.join(' ')
+
+        return '<' + name + ' ' + attr + '>' + '</' + name + '>' if self_closing_node?(node)
+        '<' + name + ' ' + attr + '>' + value + '</' + name + '>'
+      end
+
+      def adjust_name(node)
+        name = node.name.downcase
+        map = option('name_mapping')
+        map[name.to_sym] ? map[name.to_sym] : name
+      end
+
+      def contextualize(name, context)
+        if self.tokens[name] and self.tokens[name] != context
+          index = 0
+          matches = name.match(/\d+$/)
+          if matches and matches.length > 0
+            index = matches[matches.length-1].to_i
+            name = name.gsub(index.to_s, '')
+          end
+          name += (index + 1).to_s
+          return contextualize(name, context)
+        end
+
+        self.tokens[name] = context
+        name
+      end
+
+      def debug(doc)
+        self.doc = doc
+        debug_tree(self.doc, 0)
+      end
+
+      def debug_tree(node, depth)
+        padding = ('=' * (depth+1))
+
+        Tml.logger.log(padding + '=> ' + (node) + ': ' + node_info(node))
+
+        (node.children || []).each do |child|
+          debug_tree(child, depth+1)
+        end
+      end
+
+      def node_info(node)
+        info = []
+        info << node.type
+
+        info << node.tagName if node.type == 1
+
+        if inline_node?(node)
+          info << 'inline'
+          if has_inline_or_text_siblings?(node)
+            info << 'sentence'
+          else
+            info << 'only translatable'
+          end
+        end
+
+        info << 'self closing' if self_closing_node?(node)
+        info << 'only child' if only_child?(node)
+
+        return "[#{info.join(', ')}]: " + node.inner_text if node.type == 3
+        "[#{info.join(', ')}]"
+      end
+
+    end
+  end
+end