utf8_validator 1.0.13 → 1.0.14

data/utils/gencp.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+#
+#--
+# Copyright (c) 2016 Guy Allard
+#--
+#
+# Generate UTF8 Code Points
+#
+=begin
+
+From wikipediia:
+
+Bits_of_code_point	First_code_point	Last_code_point	Bytes_in_sequence	Byte_1	    Byte_2	    Byte_3	    Byte_4	   Byte_5	       Byte_6
+  7	                            U+0000	             U+007F             	1                             	0xxxxxxx
+11                           	U+0080	              U+07FF             	 2                             	110xxxxx 	10xxxxxx
+16                          	U+0800               	U+FFFF            	 3                             	1110xxxx	10xxxxxx	10xxxxxx
+21                          	U+10000             U+1FFFFF           4                             	11110xxx  10xxxxxx  10xxxxxx	10xxxxxx
+The following sequences are not part of the UTF-8 standard, only part of the original proposal
+26	                          U+200000	          U+3FFFFFF	       5                             	111110xx	10xxxxxx	10xxxxxx	10xxxxxx	10xxxxxx
+31                            U+4000000	         U+7FFFFFFF	     6                            	1111110x 	10xxxxxx	10xxxxxx  10xxxxxx 	10xxxxxx	10xxxxxx
+
+Note:  in the U+ notation the numbers are hexadecimal
+
+=end
+
+class CPData
+
+  attr_reader :name, :ubytes, :first, :last
+
+  def initialize(name, ubytes, first, last)
+    @name, @ubytes, @first, @last = name, ubytes, first, last
+  end
+end
+#
+bits7 = CPData.new("char1", 4, 0x0000, 0x007f)
+bits11 = CPData.new("char2", 4, 0x0080, 0x07ff)
+bits16a = CPData.new("char3a", 4, 0x0800, 0xcfff)
+bits16b = CPData.new("char3b", 4, 0xe000, 0xffff)
+bits21 = CPData.new("char4", 5, 0x10000, 0x1ffff)
+#
+bits26 = CPData.new("char5", 6, 0x10000, 0x1ffff)
+bits31 = CPData.new("char6", 7, 0x10000, 0x1ffff)
+bitssp = CPData.new("surrpair", 4, 0xd800, 0xdfff)
+###########
+
+def dquote(inp)
+    "\"#{inp}\""
+end
+
+pa = [bits7, bits11, bits16a, bits16b, bits21, bitssp]
+
+pa.each do |na|
+  n = 0
+  na.first.upto(na.last) do |nn|
+    n += 1
+    hs = sprintf("\\u%04x", nn)
+    puts "#{dquote(hs)},      # CPData Name: #{na.name} n:#{n}"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: utf8_validator
 version: !ruby/object:Gem::Version
-  version: 1.0.13
+  version: 1.0.14
 platform: ruby
 authors:
 - Guy Allard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-08 00:00:00.000000000 Z
+date: 2016-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jeweler
@@ -41,9 +41,12 @@ files:
 - lib/validation/errors.rb
 - lib/validation/validator.rb
 - test/helper.rb
+- test/test_code_points.rb
 - test/test_raise_request.rb
+- test/test_surrogate_half_first_point.rb
 - test/test_utf8_validator.rb
 - utf8_validator.gemspec
+- utils/gencp.rb
 homepage: http://github.com/gmallard/utf8_validator
 licenses:
 - MIT