tml 5.0.1 → 5.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5a514613d1a01ea544a896e526f38e5f9843df78
-  data.tar.gz: b1dceccffd886b61d390dbf323da3578b181835f
+  metadata.gz: 3bfad1df09bdc30e9dff8ae24cfc73dd044cc1b7
+  data.tar.gz: db29aecf3e0e719844ed5366262e843fa36cf194
 SHA512:
-  metadata.gz: f86dea985be31eaf6fcfc391615bc1287dd932704c736781834ba7d5acb029ff2f7e251893ff0593d5a1502442e62f392bc472d2f54279712661a4290e8267ed
-  data.tar.gz: 47cb409b52c980688d033f7e5b1588d1a4424d1e501f12bf564e17813b4fca668b2065ba23ccda4c24286a3a703c758c0386b1fc1b1fd45ec264c9a727e26dd1
+  metadata.gz: ec1a8dd025cd3b506f519e72550d8c891342bee71127531e8cbf00de7710f90706e2b679df58a8ce3b5a66c38f77d88a327a5fcbfa7c6063e47c663b8e4b3e90
+  data.tar.gz: b5f76143c7c34632dcd4c3193f822640916b62786b53c4a21fd8ef5dbea3fec83cbadedbf426b923e43df66e4a420a9f3adaa73f27ae91088812aa8801336efe

data/lib/tml/config.rb

@@ -114,25 +114,44 @@ module Tml
         debug_format: '{{{{$0}}}}',
         split_sentences: false,
         nodes: {
-            ignored:    [],
-            scripts:    %w(style script),
-            inline:     %w(a span i b img strong s em u sub sup),
-            short:      %w(i b),
-            splitters:  %w(br hr)
+          ignored:    %w(),
+          scripts:    %w(style script code pre),
+          inline:     %w(a span i b img strong s em u sub sup),
+          short:      %w(i b),
+          splitters:  %w(br hr)
         },
         attributes: {
-            labels:     %w(title alt)
+          labels:     %w(title alt)
         },
         name_mapping: {
-            b:    'bold',
-            i:    'italic',
-            a:    'link',
-            img:  'picture'
+          b:    'bold',
+          i:    'italic',
+          a:    'link',
+          img:  'picture'
         },
         data_tokens: {
-            special: false,
-            numeric: false,
-            numeric_name: 'num'
+          special: {
+            enable: true,
+            regex: /(&[^;]*;)/
+          },
+          date: {
+            enabled: true,
+            formats: [
+              [/((Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d+,\s+\d+)/, "{month} {day}, {year}"],
+              [/((January|February|March|April|May|June|July|August|September|October|November|December)\s+\d+,\s+\d+)/, "{month} {day}, {year}"],
+              [/(\d+\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec),\s+\d+)/, "{day} {month}, {year}"],
+              [/(\d+\s+(January|February|March|April|May|June|July|August|September|October|November|December),\s+\d+)/, "{day} {month}, {year}"]
+            ],
+            name: 'date'
+          },
+          rules: [
+            {enabled: true, name: 'time',     regex: /(\d{1,2}:\d{1,2}\s+([A-Z]{2,3}|am|pm|AM|PM)?)/},
+            {enabled: true, name: 'phone',    regex: /((\d{1}-)?\d{3}-\d{3}-\d{4}|\d?\(\d{3}\)\s*\d{3}-\d{4}|(\d.)?\d{3}.\d{3}.\d{4})/},
+            {enabled: true, name: 'email',    regex: /([-a-z0-9~!$%^&*_=+}{\'?]+(\.[-a-z0-9~!$%^&*_=+}{\'?]+)*@([a-z0-9_][-a-z0-9_]*(\.[-a-z0-9_]+)*\.(aero|arpa|biz|com|coop|edu|gov|info|int|mil|museum|name|net|org|pro|travel|io|mobi|[a-z][a-z])|([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}))(:[0-9]{1,5})?)/},
+            {enabled: true, name: 'price',    regex: /(\$\d*(,\d*)*(\.\d*)?)/},
+            {enabled: true, name: 'fraction', regex: /(\d+\/\d+)/},
+            {enabled: true, name: 'num',      regex: /(\b\d*(,\d*)*(\.\d*)?%?\b)/}
+          ]
         }
       }
 

data/lib/tml/tokenizers/dom.rb

@@ -36,10 +36,6 @@ module Tml
   module Tokenizers
     class Dom
 
-      HTML_SPECIAL_CHAR_REGEX = /(&[^;]*;)/
-      INDEPENDENT_NUMBER_REGEX = /^(\d+)$|^(\d+[.,;\s])|(\s\d+)$|(\s\d+[,;\s])/
-      VERBOSE_DATE_REGEX = /(((Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)|(January|February|March|April|May|June|July|August|September|October|November|December))\\s\\d+(,\\s\\d+)*(,*\\sat\\s\\d+:\\d+(\\sUTC))*)/
-
       attr_accessor :context, :tokens, :options
 
       def initialize(context = {}, options = {})
@@ -54,8 +50,7 @@ module Tml
 
       def translate_tree(node)
         if non_translatable_node?(node)
-          return node.children.first.inner_text if node.children.count == 1
-          return ''
+          return node.inner_html
         end
 
         return translate_tml(node.inner_text) if node.type == 3
@@ -90,10 +85,19 @@ module Tml
         html
       end
 
+      def no_translate_node?(node)
+        return unless node && node.type == 1 && node.attributes
+        node.attributes.each do |name, attribute|
+          return true if name == 'notranslate' or attribute.value.index('notranslate')
+        end
+        false
+      end
+
       def non_translatable_node?(node)
         return false unless node
         return true if node.type == 1 && (option('nodes.scripts') || []).index(node.name.downcase)
         return true if node.type == 1 && node.children.length === 0 && node.inner_text == ''
+        return true if no_translate_node?(node)
         false
       end
 
@@ -222,30 +226,55 @@ module Tml
         value.gsub(/^\s+/, '')
       end
 
-      def replace_special_characters(text)
-        return text if option('data_tokens.special')
-
-        matches = text.match(HTML_SPECIAL_CHAR_REGEX)
-        matches.each do  |match|
-          token = match[1, - 2]
-          self.context[token] = match
-          text = text.gsub(match, "{#{token}}")
-        end
-
-        text
-      end
-
       def generate_data_tokens(text)
-        return text unless option('data_tokens.numeric')
+        if option('data_tokens.special.enabled')
+          matches = text.scan(option('data_tokens.special.regex'))
+          matches.each do  |match|
+            token = match[1, - 2]
+            self.context[token] = match
+            text = text.gsub(match, "{#{token}}")
+          end
+        end
 
-        matches = text.match(INDEPENDENT_NUMBER_REGEX) || []
-        token_name = option('data_tokens.numeric_name')
+        if option('data_tokens.date.enabled')
+          token_name = option('data_tokens.date.name')
+          formats = option('data_tokens.date.formats')
+          formats.each do |format|
+            regex = format[0]
+            # date_format = format[1]
+
+            matches = text.scan(regex)
+            if matches
+              matches.each do |match|
+                next if match.first.nil? or match.first == ''
+                date = match.first
+                token = self.contextualize(token_name, date)
+                replacement = "{#{token}}"
+                text = text.gsub(date, replacement)
+              end
+            end
+          end
+        end
 
-        matches.each do |match|
-          value = match.gsub(/[.,;\s]/, '')
-          token = contextualize(token_name, value.to_i)
-          replacement = match.replace(value, "{#{token}}")
-          text = text.gsub(match, match.gsub(value, replacement))
+        rules = option('data_tokens.rules')
+        if rules
+          rules.each do |rule|
+            if rule[:enabled]
+              matches = text.scan(rule[:regex])
+
+              if matches
+                matches.each do |match|
+                  next if match.first.nil? or match.first == ''
+                  value = match.first.strip
+
+                  unless value == ''
+                    token = contextualize(rule[:name], value.gsub(/[.,;\s]/, '').to_i)
+                    text = text.gsub(value, value.gsub(value, "{#{token}}"))
+                  end
+                end
+              end
+            end
+          end
         end
 
         text

data/lib/tml/version.rb

@@ -30,5 +30,5 @@
 #++
 
 module Tml
-  VERSION = '5.0.1'
+  VERSION = '5.0.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tml
 version: !ruby/object:Gem::Version
-  version: 5.0.1
+  version: 5.0.2
 platform: ruby
 authors:
 - Michael Berkovich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-08 00:00:00.000000000 Z
+date: 2015-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday