unibits 2.1.1 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 62770bc741ec5a7693759d4e888d46cf7b3860f2
-  data.tar.gz: 171c479f1f4ebdcf11bc97c4e4e0b94bac713abb
+  metadata.gz: ca4463dcb78d345bf7cd7b89687d618803fb9a98
+  data.tar.gz: c2657ec5e6e74a70183e97ae629695170c446759
 SHA512:
-  metadata.gz: 793cdfeca34183b1b1f6cc219ddd822b9faac5542e92f1ec1065c3fd6ff07c38762becc5312d9bd29fbe3f976bd6eabb3e4c5e228bcfbe8e6340e90b5407234e
-  data.tar.gz: 9691e9072ff3084706e5dfe1ed51b1e6c206d2b11a9eaf710a37ee7b5f1e09b15a6eb3d85e3cb4517a64ef9e5b6aa0bb6bfadee72285487a4a1d0f3f66fa32bd
+  metadata.gz: 62beee1710b54962120c55b8b8bec94c42cda2305c9eab4d190945a0ec1716de5b8dae9cce3c201dc352c993c60d0709f8854b436241ab020a2c3d1348087b93
+  data.tar.gz: 0b7c3a0446cf3b0f3cb943465753fe13db6a3b130ffc9401e426300faa4be034f85c3d7af399a3c51617322ee3622d221c1e29387e6d8297c4b53cf54f1ec60d

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## CHANGELOG
 
+### 2.2.0
+
+* Add mongolian free variation selectors and combining grapheme joiner to interesting codepoints list
+* Green highlighting of "marks" in Unicode
+* Always use dotted circle for non-spacing marks
+* Always prepend enclosing marks with a space
+* Update characteristics gem (includes new blanks and UTF-8 dialects with japanese emojis)
+
 ### 2.1.1
 
 * Proper UTF-32 validness / invalid codepoint highlighting, see https://bugs.ruby-lang.org/issues/13292

data/README.md

@@ -16,6 +16,7 @@ Each byte of the given string is highlighted using the following mechanism (char
 - Blue for control characters
 - Light blue for blanks
 - Non-control formatting characters in pink
+- Green for marks (Unicode only)
 - Random color for all other characters
 
 ## Setup

data/bin/unibits

@@ -56,6 +56,7 @@ if argv[:help]
   #{Paint["control",    Unibits::COLORS[:control]]}
   #{Paint["blank",      Unibits::COLORS[:blank]]}
   #{Paint["format",     Unibits::COLORS[:format]]}
+  #{Paint["mark",       Unibits::COLORS[:mark]]}
 
   #{Paint["STATS", :underline]}
 
@@ -76,6 +77,12 @@ if argv[:help]
   incompl. | not enough bytes left to finish codepoint
   hlf.srg. | other half of surrogate missing
 
+  UTF-32
+
+  incompl. | not enough bytes left to finish codepoint
+  toolarge | codepoint value exceeds maximum allowed (U+10FFFF)
+  sur.gate | codepoint value would be a surrogate half
+
   #{Paint["MORE INFO", :underline]}
 
   https://github.com/janlelis/unibits

data/lib/unibits.rb

@@ -10,6 +10,7 @@ module Unibits
   SUPPORTED_ENCODINGS = Encoding.name_list.grep(
     Regexp.union(
       /^UTF-8$/,
+      /^UTF8-/,
       /^UTF-...E$/,
       /^ASCII-8BIT$/,
       /^US-ASCII$/,
@@ -30,6 +31,7 @@ module Unibits
     control: "#0000FF",
     blank: "#33AADD",
     format: "#FF00FF",
+    mark: "#228822",
   }
 
   DEFAULT_TERMINAL_WIDTH = 80
@@ -100,7 +102,7 @@ module Unibits
             case encoding_name
             when "US-ASCII"
               codepoint = "invalid"
-            when "UTF-8"
+            when "UTF-8", /^UTF8/
               # this tries to detect what is wrong with this utf-8 encoded string
               # sorry for this mess
               case char.unpack("B*")[0]
@@ -244,6 +246,8 @@ module Unibits
       COLORS[:blank]
     elsif char_info.format?
       COLORS[:format]
+    elsif char_info.unicode? && char_info.category[0] == "M"
+      COLORS[:mark]
     else
       random_color
     end
@@ -267,7 +271,7 @@ module Unibits
       when 'ASCII-8BIT'
         bin_byte_1 = ""
         bin_byte_2 = bin_byte_complete
-      when 'UTF-8'
+      when 'UTF-8', /^UTF8/
         if byteindex == 0
           if bin_byte_complete =~ /^(0|1{2,4}0)([01]+)$/
             bin_byte_1 = $1

data/lib/unibits/symbolify.rb

@@ -79,6 +79,10 @@ module Unibits
     }
 
     INTERESTING_CODEPOINTS = {
+      0x180B => "FVS1",
+      0x180C => "FVS2",
+      0x180D => "FVS3",
+
       0x200E => "LRM",
       0x200F => "RLM",
       0x202A => "LRE",
@@ -91,6 +95,8 @@ module Unibits
       0x2068 => "FSI",
       0x2069 => "PDI",
 
+      0x034F => "CGJ",
+
       0xFE00 => "VS1",
       0xFE01 => "VS2",
       0xFE02 => "VS3",
@@ -395,9 +401,8 @@ module Unibits
     def self.unicode(char, char_info)
       return "n/a" if !char_info.assigned?
 
-      char = char.dup
+      char = char.dup.encode("UTF-8")
       ord = char.ord
-      encoding = char_info.encoding
 
       if char_info.delete?
         char = CONTROL_DELETE_SYMBOL
@@ -406,15 +411,18 @@ module Unibits
       elsif char_info.c1?
         char = CONTROL_C1_NAMES[ord]
       elsif char_info.blank?
-        char = "]".encode(encoding) + char + "[".encode(encoding)
+        char = "]" + char + "["
       elsif ord > 917536 && ord < 917631
-        char = "TAG ".encode(encoding) +
-               char.tr(TAGS.encode(encoding), ASCII_CHARS.encode(encoding))
+        char = "TAG " + char.tr(TAGS, ASCII_CHARS)
+      elsif char_info.category == "Mn"
+        char = "◌" + char
+      elsif char_info.category == "Me"
+        char = " " + char
       else
         char = INTERESTING_CODEPOINTS[char.ord] || char
       end
 
-      char.encode("UTF-8")
+      char
     end
 
     def self.byte(char, char_info)

data/lib/unibits/version.rb

@@ -1,3 +1,3 @@
 module Unibits
-  VERSION = "2.1.1".freeze
+  VERSION = "2.2.0".freeze
 end

data/unibits.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency 'paint', '>= 0.9', '< 3.0'
   gem.add_dependency 'unicode-display_width', '~> 1.1'
-  gem.add_dependency 'characteristics', '~> 0.3.0'
+  gem.add_dependency 'characteristics', '~> 0.4'
   gem.add_dependency 'rationalist', '~> 2.0'
 
   gem.required_ruby_version = "~> 2.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unibits
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0
 platform: ruby
 authors:
 - Jan Lelis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-19 00:00:00.000000000 Z
+date: 2017-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: paint
@@ -50,14 +50,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: rationalist
   requirement: !ruby/object:Gem::Requirement