modl 0.3.19 → 0.3.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e391e57bad47d661e08c986b5a5b660fc10312e8c6a196b00318fbfe57ec621
-  data.tar.gz: df205763034d607acf635c6d2b2ce37c3b48bdf39626c17d9eead2f99f351a77
+  metadata.gz: '00617859c13359189d14d838eb6701f4a755f3933631e7771c664751e1779651'
+  data.tar.gz: 46967560c08f4eb541553bc184e0688040e91f5e3513da6709d316dae74b7f4a
 SHA512:
-  metadata.gz: 7208bfae428e5f0de6a54f0727f525f2db44a11c2022dc493e5696a5cfaff50b3a45722f2a657ee93f242eae670e273071a3e137dd70859777810120c1751bc1
-  data.tar.gz: ac22712bed4d0d4dd9ac46b129136f1617d381367ddbe6dc0c47173ecc0ef22bc110b08fbdccfa515e818834ab7ebf70d13c19bc241f4b946d897013e2f28dd2
+  metadata.gz: cb09f90f72233e3de2b672aca0c1d3ef04f56ced05440a6a37bd31eb029bf6e23b46e4e880b859dbce4d5ba1a162b8d3841edb5c04d7ed7fa7b67edb06db4ab2
+  data.tar.gz: 2740782db327af3741d4627fccce8d4af0875830d007171df7f9b705fab0faecd8fe3ccdb820d601a7079f78f9a838ab295b0acbc7f64f322b1eb6a2c97b4cb5

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+0.3.20
+===
+- Support escaping unicode escape sequences so they are not replaced by unicode characters.
+
 0.3.19
 ===
 - Fix bug with `*assign` when there is only 1 item in the array.

data/grammar_tests/base_tests.json

@@ -3014,12 +3014,12 @@
   },
   {
     "id": "316",
-    "input": "We\\\\u2019re=We\\\\u2019re;You~u2019re=We~u2019re",
+    "input": "We\\u2019re=We\\u2019re;You~u2019re=We~u2019re",
     "expected_output": "{\n  \"We’re\": \"We’re\",\n  \"You’re\": \"We’re\"\n}",
     "tested_features": [
       "unicode"
     ],
-    "minimised_modl": "We\\\\u2019re=We\\\\u2019re;You~u2019re=We~u2019re"
+    "minimised_modl": "We\\u2019re=We\\u2019re;You~u2019re=We~u2019re"
   },
   {
     "id": "317",
@@ -3056,5 +3056,35 @@
       "class"
     ],
     "minimised_modl": "*class(*id=m;*name=message;*superclass=map;*assign=[[direction;date_time;message]];method=sms);[m=out:2018-03-22 15\\:25:Hi;m=in:2018-03-22 15\\:26:Hello, how are you?;m=out:2018-03-22 15\\:25:Hi, good thanks;m=out:2018-03-22 15\\:26:How about you?;m=in:2018-03-22 1\\:26:Yes, fine thanks. What are you up to?;m=out:2018-03-22 15\\:25:Just testing out MODL;m=in:2018-03-22 15\\:26:Cool!]"
+  },
+  {
+    "id": "321",
+    "input": "test=\\~u2019",
+    "expected_output": "{\n    \"test\": \"~u2019\"\n}",
+    "tested_features": [
+      "unicode",
+      "escapes"
+    ],
+    "minimised_modl": "test=\\~u2019"
+  },
+  {
+    "id": "322",
+    "input": "test=\\\\u2019",
+    "expected_output": "{\n    \"test\": \"\\\\u2019\"\n}",
+    "tested_features": [
+      "unicode",
+      "escapes"
+    ],
+    "minimised_modl": "test=\\\\u2019"
+  },
+  {
+    "id": "323",
+    "input": "test=\\\\~u2019\\\\u2019~u2019\\\\u2019",
+    "expected_output": "{\n    \"test\": \"~u2019\\\\u2019\u2019\\\\u2019\"\n}",
+    "tested_features": [
+      "unicode",
+      "escapes"
+    ],
+    "minimised_modl": "test=\\\\~u2019\\\\u2019~u2019\\\\u2019"
   }
 ]

data/lib/modl/parser/evaluator.rb

@@ -54,7 +54,7 @@ module MODL
         while i < condition.values.length
           item = condition.values[i]
           if item.primitive.constant
-            value2 = Substitutions.process(item.text)
+            value2 = Substitutions.process UnicodeEscapes.process(item.text)
           else
             value2, success = value(global, item.text)
           end
@@ -105,12 +105,12 @@ module MODL
           if ikey.to_s == key
             index_val = global.index[ikey]
             value1 = index_val.respond_to?(:text) ? index_val.text : nil
-            value1 = Substitutions.process(value1)
+            value1 = Substitutions.process UnicodeEscapes.process(value1)
           else
             pair = global.pair(key)
-            return Substitutions.process(k) unless pair
+            return Substitutions.process UnicodeEscapes.process(k) unless pair
 
-            value1 = Substitutions.process(pair.text)
+            value1 = Substitutions.process UnicodeEscapes.process(pair.text)
           end
           success = true
         end

data/lib/modl/parser/parsed.rb

@@ -25,6 +25,7 @@
 require 'modl/parser/MODLParserBaseListener'
 require 'modl/parser/global_parse_context'
 require 'modl/parser/ref_processor'
+require 'modl/parser/unicode_escapes'
 require 'modl/parser/substitutions'
 require 'modl/parser/file_importer'
 require 'antlr4/runtime/parse_cancellation_exception'
@@ -287,7 +288,7 @@ module MODL
           return if @type == 'allow'
           return if @type == 'expect'
 
-          formatted_key = Substitutions.process @key
+          formatted_key = Substitutions.process UnicodeEscapes.process @key
           {formatted_key => @text}
         end
 
@@ -667,14 +668,14 @@ module MODL
             if user_method
               return user_method.run(@string.string)
             end
-            return StandardMethods.run_method(key, Substitutions.process(@string.string))
+            StandardMethods.run_method(key, Substitutions.process(UnicodeEscapes.process(@string.string)))
           end
         end
 
         def extract_hash
           result, _ignore = RefProcessor.deref(@text, @global) unless @constant
           result = @text if @constant
-          Substitutions.process result
+          Substitutions.process UnicodeEscapes.process result
         end
 
         def evaluate
@@ -743,7 +744,7 @@ module MODL
         end
 
         def extract_hash
-          @string = Substitutions.process @string
+          @string = Substitutions.process UnicodeEscapes.process @string
         end
       end
 

data/lib/modl/parser/substitutions.rb

@@ -90,19 +90,7 @@ module MODL
             break unless new_str && new_str != prev
           end
         end
-        convert_unicode new_str
-      end
-
-      def self.convert_unicode(s)
-        uni_str_idx = s.index('\u')
-        uni_str_idx = s.index('~u') if uni_str_idx.nil?
-        return s if uni_str_idx.nil?
-
-        value = s.slice(uni_str_idx + 2, 4).to_i(16)
-        uni_str = s.slice(uni_str_idx, 6)
-        uni_val = value.chr(Encoding::UTF_8)
-        result = s.sub(uni_str, uni_val)
-        return convert_unicode result
+        new_str
       end
     end
   end

data/lib/modl/parser/unicode_escapes.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 NUM Technology Ltd
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module MODL
+  module Parser
+    # Escape-sequence replacements for MODL files.
+    class UnicodeEscapes
+
+      # Replace all unicode escape sequences in the supplied string and return the new value.
+      def self.process(str)
+        return str unless str.is_a? String
+
+        start = 0
+        result = str
+        loop do
+
+          backslash_u = result.index('\u', start)
+          tilde_u = result.index('~u', start)
+
+          break if tilde_u.nil? && backslash_u.nil?
+
+          if tilde_u.nil?
+            uni_str_idx = backslash_u
+          elsif backslash_u.nil?
+            uni_str_idx = tilde_u
+          else
+            uni_str_idx = [backslash_u, tilde_u].min
+          end
+
+          break if uni_str_idx + 6 > result.length
+
+          start = uni_str_idx + 1
+
+          next if uni_str_idx > 0 && result[uni_str_idx - 1] == '~'
+          next if uni_str_idx > 0 && result[uni_str_idx - 1] == '\\'
+
+          value = result.slice(uni_str_idx + 2, 4).to_i(16)
+          uni_val = value.chr(Encoding::UTF_8)
+          left = result.slice(0, uni_str_idx)
+          right = result.slice(uni_str_idx + 6, result.length)
+          result = left + uni_val + right unless right.nil?
+          result = left + uni_val if right.nil?
+        end
+        result
+      end
+    end
+  end
+end

data/lib/modl/parser/version.rb

@@ -24,6 +24,6 @@
 
 module MODL
   module Parser
-    VERSION = "0.3.19"
+    VERSION = "0.3.20"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: modl
 version: !ruby/object:Gem::Version
-  version: 0.3.19
+  version: 0.3.20
 platform: ruby
 authors:
 - Tony Walmsley
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-11 00:00:00.000000000 Z
+date: 2019-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -131,6 +131,7 @@ files:
 - lib/modl/parser/substitutions.rb
 - lib/modl/parser/sutil.rb
 - lib/modl/parser/throwing_error_listener.rb
+- lib/modl/parser/unicode_escapes.rb
 - lib/modl/parser/version.rb
 - modl.gemspec
 homepage: https://github.com/MODLanguage/ruby-interpreter