tml 5.6.5 → 5.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c34d0e6209b1b34c4143f0b16ecbc73aec7a5324
-  data.tar.gz: 20f9c409c786a9a67cb76e74eb3cdbe285f9eb3a
+  metadata.gz: a26bbf3c59deca21956a94bc33d811377797034e
+  data.tar.gz: fdd19956dcf43dc02228c8c43ea8687a8eb47d34
 SHA512:
-  metadata.gz: 8ee0f8b41b9d66d8d1f05172df2318eade8f05c408f299137fb45e791b3cd03b0ac20da55e8c28cda300faa08c736c8f0d9dfdec49fbae47dfc4094cef5bf2dc
-  data.tar.gz: 690f0c2581fcdc822a8e63c45265fdcce2eb2a4b9298a700dd9845b9429ef1911395acf1995946bbcd9aa7d91764e638277db9dc404d438421c16c9c3334e2af
+  metadata.gz: f5277433d7a88c672d46f9c214add49e06bb52752fd1e9f23b06f8c1fad3f0db9c40723c3d159078644b59ecfd8f6d372c7b8665f8e6781404788cf19b7ffd7c
+  data.tar.gz: 3e31327a1b93feffc6f86e1d974c0ba64e0a90aac225a154d18eb8c7de67316f1bf10438db03a88e40fe47c4a38fca0bbf6779e4ede7a5c77f8c70855e4755b0

data/lib/tml.rb

@@ -32,7 +32,9 @@
 
 module Tml
   module Api end
-  module Tokens end
+  module Tokens
+    module XMessage end
+  end
   module Tokenizers end
   module Rules end
   module Decorators end
@@ -64,7 +66,7 @@ module Tml
   end
 end
 
-%w(tml/base.rb tml tml/api tml/rules_engine tml/tokens tml/tokenizers tml/decorators tml/cache_adapters tml/cache tml/ext).each do |f|
+%w(tml/base.rb tml tml/api tml/rules_engine tml/tokens tml/tokens/x_message tml/tokenizers tml/decorators tml/cache_adapters tml/cache tml/ext).each do |f|
   if f.index('.rb')
     require(File.expand_path(File.join(File.dirname(__FILE__), f)))
     next

data/lib/tml/application.rb

@@ -237,7 +237,7 @@ class Tml::Application < Tml::Base
       source = Tml::Source.new(:source => source_key, :application => self)
       params << {
           :source => source_key,
-          :keys => keys.values.collect{|tkey| tkey.to_hash(:label, :description, :locale, :level)}
+          :keys => keys.values.collect{|tkey| tkey.to_hash(:label, :description, :locale, :level, :syntax)}
       }
       source.reset_cache
     end
@@ -307,6 +307,8 @@ class Tml::Application < Tml::Base
 
   # Cache translations within application object
   def cache_translations(locale, key, new_translations)
+    return if new_translations.nil?
+
     self.translations ||= {}
     self.translations[locale] ||= {}
     self.translations[locale][key] = new_translations.collect do |t|

data/lib/tml/config.rb

@@ -242,6 +242,7 @@ module Tml
             :trade  =>  '™',       # TM
           },
           :decoration => {
+            :anchor =>  "<a href='{$href}'>{$0}</a>",
             :strong =>  '<strong>{$0}</strong>',
             :bold   =>  '<strong>{$0}</strong>',
             :b      =>  '<strong>{$0}</strong>',
@@ -280,6 +281,7 @@ module Tml
             :trade  =>  '™',
           },
           :decoration => {
+            :anchor =>  '{$0}',
             :strong =>  '{$0}',
             :bold   =>  '{$0}',
             :b      =>  '{$0}',
@@ -409,6 +411,32 @@ module Tml
     #  @default_level
     #end
 
+    def xmessage_rule_key_mapping
+      @rule_key_mapping ||= {
+          number: {
+              one: 'singular',
+              few: 'few',
+              many: 'many',
+              other: 'plural'
+          },
+          gender: {
+              male: 'male',
+              female: 'female',
+              neutral: 'neutral',
+              other: 'other',
+          },
+          date: {
+              future: 'future',
+              present: 'present',
+              past: 'past'
+          }
+      }
+    end
+
+    def xmessage_decoration_tokens
+      @xmessage_tokens ||= @default_tokens[:html][:decoration].keys
+    end
+
     def default_locale
       @locale[:default] || 'en'
     end

data/lib/tml/language.rb

@@ -149,14 +149,17 @@ class Tml::Language < Tml::Base
       :description  => params[:description],
       :locale       => hash_value(params[:options], :locale) || Tml.session.block_option(:locale) || Tml.config.default_locale,
       :level        => hash_value(params[:options], :level) || Tml.session.block_option(:level) || Tml.config.default_level,
+      :syntax       => hash_value(params[:options], :syntax),
       :translations => []
     })
 
-    # pp "Translating #{params[:label]} from: #{translation_key.locale.inspect} to #{locale.inspect}"
+    # pp "Translating #{params[:label]} from: #{translation_key.locale.inspect} to #{locale.inspect} with syntax #{translation_key.syntax}"
     # Tml.logger.info("Translating #{params[:label]} from: #{translation_key.locale.inspect} to #{locale.inspect}")
 
     params[:tokens] ||= {}
-    params[:tokens][:viewing_user] ||= Tml.session.current_user
+    if params[:tokens].is_a?(Hash)
+      params[:tokens][:viewing_user] ||= Tml.session.current_user
+    end
 
     if Tml.config.disabled? or application.nil?
       return translation_key.substitute_tokens(params[:label], params[:tokens], self, params[:options]).tml_translated
@@ -212,6 +215,9 @@ class Tml::Language < Tml::Base
     end
 
     translation_key.translate(self, params[:tokens], params[:options]).tml_translated
+  rescue Exception => ex
+    pp ex, ex.backtrace
+    return params[:label]
   end
   alias :tr :translate
 

data/lib/tml/tokenizers/data.rb

@@ -54,11 +54,11 @@ module Tml
       attr_accessor :text, :context, :tokens, :opts
 
       def self.supported_tokens
-        [Tml::Tokens::Data, Tml::Tokens::Method, Tml::Tokens::Transform]
+        [Tml::Tokens::Data, Tml::Tokens::Method, Tml::Tokens::Transform, Tml::Tokens::Map]
       end
 
       def self.required?(label)
-        label.index("{")
+        label.index('{')
       end
 
       def initialize(text, context={}, opts={})
@@ -84,10 +84,12 @@ module Tml
 
       def substitute(language, options = {})
         label = self.text
+
         tokens.each do |token|
           next unless token_allowed?(token)
           label = token.substitute(label, context, language, options)
         end
+
         label
       end
 

data/lib/tml/tokenizers/decoration.rb

@@ -53,7 +53,7 @@ module Tml
   module Tokenizers
     class Decoration
 
-      attr_reader :tokens, :fragments, :context, :text, :opts
+      attr_reader :tokens, :fragments, :context, :label, :opts
 
       RESERVED_TOKEN       = 'tml'
 
@@ -63,14 +63,14 @@ module Tml
       RE_LONG_TOKEN_END    = '\[\/[\w]*\]'                     # [/link]
       RE_HTML_TOKEN_START  = '<[^\>]*>'                        # <link>
       RE_HTML_TOKEN_END    = '<\/[^\>]*>'                      # </link>
-      RE_TEXT              = '[^\[\]<>]+'                        # '[\w\s!.:{}\(\)\|,?]*'
+      RE_TEXT              = '[^\[\]<>]+'                      # '[\w\s!.:{}\(\)\|,?]*'
 
       def self.required?(label)
         label.index('[') or label.index('<')
       end
 
-      def initialize(text, context = {}, opts = {})
-        @text = "[#{RESERVED_TOKEN}]#{text}[/#{RESERVED_TOKEN}]"
+      def initialize(label, context = {}, opts = {})
+        @label = label
         @context = context
         @opts = opts
         tokenize
@@ -84,12 +84,12 @@ module Tml
               RE_HTML_TOKEN_START,
               RE_HTML_TOKEN_END,
               RE_TEXT].join('|')
-        @fragments = text.scan(/#{re}/)
+        @fragments = "[#{RESERVED_TOKEN}]#{@label}[/#{RESERVED_TOKEN}]".scan(/#{re}/)
         @tokens = []
       end
 
       def parse
-        return @text unless fragments
+        return @label unless fragments
         token = fragments.shift
 
         if token.match(/#{RE_SHORT_TOKEN_START}/)
@@ -136,66 +136,9 @@ module Tml
         tree
       end
 
-      def default_decoration(token_name, token_value)
-        default_decoration = Tml.config.default_token_value(normalize_token(token_name), :decoration)
-
-        unless default_decoration
-          Tml.logger.error("Invalid decoration token value for #{token_name} in #{text}")
-          return token_value
-        end
-
-        default_decoration = default_decoration.clone
-        decoration_token_values = context[token_name.to_sym] || context[token_name.to_s]
-
-        default_decoration.gsub!('{$0}', token_value.to_s)
-
-        if decoration_token_values.is_a?(Hash)
-          decoration_token_values.keys.each do |key|
-            default_decoration.gsub!("{$#{key}}", decoration_token_values[key].to_s)
-          end
-        end
-
-        # remove unused attributes
-        default_decoration = default_decoration.gsub(/\{\$[^}]*\}/, '')
-        default_decoration
-      end
-
-      def allowed_token?(token)
-        return true if opts[:allowed_tokens].nil?
-        opts[:allowed_tokens].include?(token)
-      end
-
-      def apply(token, value)
-        return value if token == RESERVED_TOKEN
-        return value unless allowed_token?(token)
-
-        method = context[token.to_sym] || context[token.to_s]
-
-        if method
-          if method.is_a?(Proc)
-            return method.call(value)
-          end
-
-          if method.is_a?(Array) or method.is_a?(Hash)
-            return default_decoration(token, value)
-          end
-
-          if method.is_a?(String)
-            return method.to_s.gsub('{$0}', value)
-          end
-
-          return value
-        end
-
-        if Tml.config.default_token_value(normalize_token(token), :decoration)
-          return default_decoration(token, value)
-        end
-
-        value
-      end
-
-      def normalize_token(name)
-        name.to_s.gsub(/(\d)*$/, '')
+      def apply(token_name, value)
+        token = ::Tml::Tokens::Decoration.new(label, token_name)
+        token.apply(context, value, opts[:allowed_tokens])
       end
 
       def evaluate(expr)
@@ -211,7 +154,7 @@ module Tml
       end
 
       def substitute
-        evaluate(parse).gsub('[/tml]', '')
+        evaluate(parse).gsub(/[<\[]\/tml[>\]]/, '')
       end
 
     end

data/lib/tml/tokenizers/dom.rb

@@ -103,7 +103,7 @@ module Tml
 
       def translate_tml(tml)
         return tml if empty_string?(tml)
-        pp tml
+        # pp tml
 
         tml = generate_data_tokens(tml)
 

data/lib/tml/tokenizers/x_message.rb

@@ -0,0 +1,490 @@
+# encoding: UTF-8
+#--
+# Copyright (c) 2016 Translation Exchange, Inc
+#
+#  _______                  _       _   _             ______          _
+# |__   __|                | |     | | (_)           |  ____|        | |
+#    | |_ __ __ _ _ __  ___| | __ _| |_ _  ___  _ __ | |__  __  _____| |__   __ _ _ __   __ _  ___
+#    | | '__/ _` | '_ \/ __| |/ _` | __| |/ _ \| '_ \|  __| \ \/ / __| '_ \ / _` | '_ \ / _` |/ _ \
+#    | | | | (_| | | | \__ \ | (_| | |_| | (_) | | | | |____ >  < (__| | | | (_| | | | | (_| |  __/
+#    |_|_|  \__,_|_| |_|___/_|\__,_|\__|_|\___/|_| |_|______/_/\_\___|_| |_|\__,_|_| |_|\__, |\___|
+#                                                                                        __/ |
+#                                                                                       |___/
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+#######################################################################
+#
+# Decoration Token Forms:
+#
+# [link: click here]
+# or
+# [link] click here [/link]
+#
+# Decoration Tokens Allow Nesting:
+#
+# [link: {count} {_messages}]
+# [link: {count||message}]
+# [link: {count||person, people}]
+# [link: {user.name}]
+#
+#######################################################################
+
+module Tml
+  module Tokenizers
+    class XMessage
+
+      attr_accessor :label, :pos, :len, :last, :options, :tree
+
+      def optional_style_format_types
+        @optional_style_format_types ||= {
+            'text' => true,
+            'date' => true,
+            'time' => true,
+            'number' => true,
+            'name' => true,
+            'list' => true,
+            'possessive' => true,
+            'salutation' => true
+        }
+      end
+
+      def initialize(text, opts = {})
+        @label = text
+        @pos = 0
+        @len = @label ? @label.length : 0
+        @last = nil
+        @options = opts || {}
+        @tree = nil
+        tokenize
+      end
+
+      def update_last
+        @last = @pos > 0 ? @label[@pos - 1] : nil
+      end
+
+      def next_char
+        return if @len == 0 || @pos >= @len
+        update_last
+        @pos += 1
+        @label[@pos - 1]
+      end
+
+      def peek_char
+        return if @len == 0
+        @label[@pos]
+      end
+
+      def revert
+        if (@pos > 0)
+          @pos -= 1
+          update_last
+        end
+      end
+
+      def escaped?
+        @last && @last == '\\'
+      end
+
+      def no_format_style(result, c, argument_index, format_type)
+        raise "no format style allowed for format type '" + format_type + "'";
+      end
+
+      def collection_format_style(result, c, argument_index, format_type)
+        # register the format element
+        styles = []
+        subtype = 'text'; # default
+
+        if c == ','
+          # we have a sub-type
+          subtype = ''
+          c = next_char
+          while c && !',}'.index(c)
+            subtype += c
+            c = next_char
+            unless c
+              raise "expected ',' or '}', but found end of string"
+            end
+          end
+        end
+
+        result << {index: argument_index, type: format_type, subtype: subtype, styles: styles}
+
+        if c == '}'
+          return
+        end
+
+        # parse format style
+        while c
+          c = next_char
+          unless c
+            raise "expected '}', '|' or format style value, but found end of string"
+          end
+
+          if c == '}' && !escaped?
+            return
+          elsif c == '|'
+            next
+          end
+
+          style_key = ''
+          while c && !'#<|}'.index(c)
+            style_key += c
+            c = next_char
+            unless c
+              raise "expected '#', '<' or '|', but found end of string"
+            end
+          end
+
+          if c == '<'
+            style_key += c
+          end
+
+          items = []
+          styles << {key: style_key, items: items}
+
+          if '#<'.index(c)
+            traverse_text(items)
+          elsif '|}'.index(c)
+            # we found a key without value e.g. {0,param,possessive} and {0,param,prefix#.|possessive}
+            revert
+          end
+        end
+      end
+
+      def text_format_style(result, c, argument_index, format_type)
+        # parse format style
+        buffer = ''
+        c = next_char
+        unless c
+          raise "expected format style or '}', but found end of string"
+        end
+
+        while c
+          if c == '}'
+            result << {index: argument_index, type: format_type, value: buffer}
+            return
+          end
+
+          # keep adding to buffer
+          buffer += c
+          c = next_char
+          unless c
+            raise "expected '}', but found end of string"
+          end
+        end
+      end
+
+      def default_format_style(result, c, argument_index, format_type)
+        # register the format element
+        styles = []
+        result << {index: argument_index, type: format_type, styles: styles}
+
+        # parse format style
+        while c
+          c = next_char
+          unless c
+            raise "expected '}', '|' or format style value, but found end of string"
+          end
+
+          if c == '}' && !escaped?
+            return
+          elsif c == '|'
+            next
+          end
+
+          style_key = ''
+          while c && !'#<+|}'.index(c)
+            style_key += c
+            c = next_char
+            unless c
+              raise "expected '#', '<', '+' or '|', but found end of string"
+            end
+          end
+
+          if c == '<' || c == '+'
+            style_key += c
+          end
+
+          items = []
+          styles << {key: style_key, items: items}
+
+          if '#<+'.index(c)
+            traverse_text(items)
+          elsif '|}'.index(c)
+            # we found a key without value e.g. {0,param,possessive} and {0,param,prefix#.|possessive}
+            revert
+          end
+        end
+      end
+
+      def traverse_format_element(result)
+        argument_index = -1
+        format_type = nil
+        c = next_char
+
+        unless c
+          raise 'expected place holder index, but found end of string'
+        end
+
+        if c.match(/[\d:]/)
+          # process argument index
+          is_keyword = c == ':'
+          index = ''
+          while c && !',}'.index(c)
+            index += c
+            c = next_char
+            unless c
+              raise "expected ',' or '}', but found end of string";
+            end
+          end
+
+          if !is_keyword && !index.match(/\d+/)
+            throw "argument index must be numeric: #{index}"
+          end
+
+          argument_index = is_keyword ? index : index * 1
+        end
+
+        if c != '}'
+          # process format type
+          format_type = ''
+          c = next_char
+          unless c
+            raise 'expected format type, but found end of string'
+          end
+
+          while c && !',}'.index(c) && !escaped?
+            format_type += c
+            c = next_char
+            unless c
+              raise "expected ',' or '}', but found end of string"
+            end
+          end
+        end
+
+        if c == '}' && !escaped?
+          if format_type && optional_style_format_types[format_type]
+            # we found {0,number} or {0,possessive} or {0,salutation}, which are valid expressions
+            result << {type: format_type, index: argument_index}
+          else
+            if format_type
+              # we found something like {0,<type>}, which is invalid.
+              raise "expected format style for format type '#{format_type}'"
+            end
+
+            # push param format element
+            result << {type: 'param', index: argument_index}
+          end
+        elsif c == ','
+          processors = {
+              list: 'collection_format_style',
+              date: 'text_format_style',
+              time: 'text_format_style',
+              number: 'text_format_style',
+              suffix: 'text_format_style',
+              possessive: 'no_format_style',
+              salutation: 'no_format_style',
+              default: 'default_format_style'
+          }
+          processor = (processors[format_type.to_sym] || processors[:default])
+          self.send(processor, result, c, argument_index, format_type)
+        else
+          raise "expected ',' or '}', but found '#{c}' at position #{@pos}"
+        end
+      end
+
+      def traverse_text(result)
+        in_quoted_string = false
+        buffer = ''
+        c = next_char
+
+        while c do
+          if c == "'"
+            in_quoted_string = !in_quoted_string
+          end
+
+          if !in_quoted_string && c == '{' && !escaped?
+            unless buffer.empty?
+              result << {type: 'trans', value: buffer}
+              buffer = ''
+            end
+            traverse_format_element(result)
+          elsif !in_quoted_string && (c == '|' || c == '}') && !escaped?
+            revert
+            break
+          else
+            buffer += c
+          end
+          c = next_char
+        end
+
+        unless buffer.empty?
+          result << {type: 'trans', value: buffer}
+          buffer = ''
+        end
+
+        result
+      end
+
+      def tokenize
+        result = []
+        traverse_text(result)
+        @tree = result
+      rescue Exception => ex
+        pp ex
+        pp "Failed to parse the expression: " + @label
+        @tree = nil
+      end
+
+      def rule_key(context_key, rule_key)
+        return rule_key unless Tml.config.xmessage_rule_key_mapping[context_key.to_sym]
+        Tml.config.xmessage_rule_key_mapping[context_key.to_sym][rule_key.to_sym] || rule_key
+      end
+
+      def choice(language, token, token_object)
+        return unless token
+
+        context_key = token.context_keys.first
+        return unless context_key
+
+        ctx = language.context_by_keyword(context_key)
+        return unless ctx
+
+        # pp context_key, token_object
+
+        rule = ctx.find_matching_rule(token_object)
+        if rule
+          # pp context_key, rule.keyword
+          return rule_key(context_key, rule.keyword)
+        end
+
+        nil
+      end
+
+      def data?(type)
+        return false unless type
+        %w(param number).include?(type)
+      end
+
+      def decoration?(type)
+        return false unless type
+        Tml.config.xmessage_decoration_tokens.include?(type.to_sym)
+      end
+
+      def choice?(type)
+        type == 'choice'
+      end
+
+      def map?(type)
+        type == 'map'
+      end
+
+      def compile(language, exp, buffer, params)
+        style = nil
+
+        exp.each do |el|
+          token = token_by_type(el[:type], el)
+          token_object = get_token_object(params, token)
+          token_value = get_token_value(token_object, token, language)
+
+          if el[:styles]
+            if choice?(el[:type])
+              key = choice(language, token, token_object)
+              style = el[:styles].find{ |style|
+                style[:key] == key
+              }
+              if style
+                compile(language, style[:items], buffer, params)
+              end
+            elsif map?(el[:type])
+              style = el[:styles].find{ |style|
+                style[:key] == token_value
+              }
+              compile(language, style[:items], buffer, params)
+            elsif decoration?(el[:type])
+              buffer << token.open_tag(token_object)
+              compile(language, el[:styles][0][:items], buffer, params)
+              buffer << token.close_tag
+            else
+              compile(language, el[:styles][0][:items], buffer, params)
+            end
+          elsif data?(el[:type])
+            buffer << token_value
+          else
+            buffer << el[:value]
+          end
+        end
+
+        buffer
+      end
+
+      def tokens
+        @tokens ||= begin
+          tokens = []
+          extract_tokens(tree, tokens)
+          tokens.uniq{ |t| [t.class.name, t.full_name] }
+        end
+      end
+
+      def token_by_type(type, data)
+        if decoration?(type)
+          Tml::Tokens::XMessage::Decoration.new(label, data)
+        elsif data?(type)
+          Tml::Tokens::XMessage::Data.new(label, data)
+        elsif choice?(type)
+          Tml::Tokens::XMessage::Choice.new(label, data)
+        elsif map?(type)
+          return Tml::Tokens::XMessage::Map.new(label, data)
+        else
+          nil
+        end
+      end
+
+      def extract_tokens(tree, tokens)
+        tree.each do |fragment|
+          token = token_by_type(fragment[:type], fragment)
+          tokens << token if token
+          if fragment[:items]
+            extract_tokens(fragment[:items], tokens)
+          elsif fragment[:styles]
+            extract_tokens(fragment[:styles], tokens)
+          end
+        end
+      end
+
+      def get_token_object(token_values, token)
+        return nil unless token
+        token.token_object(token_values)
+      end
+
+      def get_token_value(token_object, token, language)
+        return nil unless token_object && token
+        token.token_value(token_object, language)
+      end
+
+      def substitute(language, tokens = {}, options = {})
+        return @label unless tree
+        compile(language, tree, [], tokens).join('')
+      end
+
+    end
+  end
+end