modl 0.3.22 → 0.3.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15424e85da367ff2ab9aaf7719c781891382d0b06e8f043b4c53a92da037aa7a
-  data.tar.gz: 68d4ccbd3516afd9b628aacdace4689f46706d2db151ef70b2725c3724a7cee0
+  metadata.gz: b6e08595e87f2b8fccd3ab9abfde857bd268bf2ca1c8fa1b00069d81c5644701
+  data.tar.gz: c725b89c8d561e3c84404f55d74c94d1fdf605474aa58c2fb5bc255255d7a816
 SHA512:
-  metadata.gz: 2026dcc8197b0ac9beb78555aaa2ed13817b7005a781a85fa7255b29042be14c92aa4aa455522ca67658aa92a86bd49f4e98f0b723359272b13a541dd53be496
-  data.tar.gz: 3df4f4c19bdc33b7453146c81802396c15fc90c3e9ebf073f4e8920bb128cf5e1cb7a10c50eaebf8be4631643c3ba73bcb34d031d87643e6806160b37ac2fcba
+  metadata.gz: 17d4f8fc520fb5cc16737ab869a52b1ee532b5050dc404153da7dd6c6f71f79befd6aaaddaf1857b41e317be763ca404851d8419d254041397f2cd6290ee1957
+  data.tar.gz: 6ba10d6e62a742b0c7ec38c2a82ab8f9231168a7d9ed8102ea45fe3a0f45209e2f753f980551de919eba18d7762c4dfc0ca1b3ee0530de40cfb7a90fed15c26b

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+0.3.23
+===
+- Add support for 5- and 6-digit unicode escape sequences.
+
 0.3.22
 ===
 - Two bug fixes - quoted strings, and incomplete strings for certain output situations.

data/grammar_tests/base_tests.json

@@ -3160,5 +3160,15 @@
       "object_ref"
     ],
     "minimised_modl": "_C=gb;_L=en;*load=\"http://modules.num.uk/1/rcf.txt!\";o(c[fb=abc;tw=abc])"
+  },
+  {
+    "id": "332",
+    "input": "test=~u1f60000",
+    "expected_output": "{\n    \"test\": \"\uD83D\uDE0000\"\n}",
+    "tested_features": [
+      "unicode",
+      "escapes"
+    ],
+    "minimised_modl": "test=\\~u1f60000"
   }
 ]

data/lib/modl/parser/unicode_escapes.rb

@@ -55,15 +55,58 @@ module MODL
           next if uni_str_idx > 0 && result[uni_str_idx - 1] == '~'
           next if uni_str_idx > 0 && result[uni_str_idx - 1] == '\\'
 
-          value = result.slice(uni_str_idx + 2, 4).to_i(16)
+          (value, len) = try_parse(result, uni_str_idx + 2)
           uni_val = value.chr(Encoding::UTF_8)
           left = result.slice(0, uni_str_idx)
-          right = result.slice(uni_str_idx + 6, result.length)
+          right = result.slice(uni_str_idx + (2 + len), result.length)
           result = left + uni_val + right unless right.nil?
           result = left + uni_val if right.nil?
         end
         result
       end
+
+      def self.enough_digits?(str, idx, n)
+        i = 0
+        chars = str.chars
+        while i < n && (idx + i) < str.length
+          c = chars[idx + i]
+          unless (('0'..'9').include? c) || (('a'..'f').include? c) || (('A'..'F').include? c)
+            return false
+          end
+          i += 1
+        end
+        (i == n) ? true : false
+      end
+
+      # unicode can be 4 to 6 characters
+      def self.try_parse(str, idx)
+        if enough_digits?(str, idx, 6)
+          value = str.slice(idx, 6).to_i(16)
+          if valid_range? value
+            return [value, 6]
+          end
+        end
+        if enough_digits?(str, idx, 5)
+          value = str.slice(idx, 5).to_i(16)
+          if valid_range? value
+            return [value, 5]
+          end
+        end
+        if enough_digits?(str, idx, 4)
+          value = str.slice(idx, 4).to_i(16)
+          if valid_range? value
+            return [value, 4]
+          end
+          if valid_range? value
+            return [value, 4]
+          end
+        end
+        [0, 4]
+      end
+
+      def self.valid_range?(value)
+        return ((0x100000..0x10ffff).include? value) || ((0x10000..0xfffff).include? value) || ((0..0xd7ff).include? value) || ((0xe000..0xffff).include? value) ? true : false
+      end
     end
   end
 end

data/lib/modl/parser/version.rb

@@ -24,6 +24,6 @@
 
 module MODL
   module Parser
-    VERSION = "0.3.22"
+    VERSION = "0.3.23"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: modl
 version: !ruby/object:Gem::Version
-  version: 0.3.22
+  version: 0.3.23
 platform: ruby
 authors:
 - Tony Walmsley
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-29 00:00:00.000000000 Z
+date: 2019-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake