unibits 2.4.0 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e9d89e598970e15c646b6d3dce0969afe24204fe
-  data.tar.gz: '0473794df2d445d3d0bc5d6f5c697d0c148de119'
+  metadata.gz: 89ef5ce08435a61d17d624f01b9355753446667c
+  data.tar.gz: 2c758d25836c61613a0ab1c60f8196b93421c979
 SHA512:
-  metadata.gz: f9bd47214da917798bb5e40ad7bdaf80d93a0fd2b07d43f825b8819fb81eca54935f4eff5ae3b508a8213895366a2898b049b558e368c6e69b275f7a3a86bb8f
-  data.tar.gz: 71c2583c9935c7ceead69c8d7c694af2f621bc08f952c896d0e24baf8688d747724315e1fe178e98c1e0e2138d830306e3623804ad9dd37b81c100ffa7394e19
+  metadata.gz: fc2087917695cd6855477ddaa7fa973da3c188a814564e13d7331ee4c3744aa7338efef1397b3ed6fe3a9b502522dbcace5b7d7eb20f409ecac74a56ec74d200
+  data.tar.gz: 5428461b0a77d91f746bfc565072f9a0baf126b0d7ce40d88113b54e56de4678a209bf01a03ff2fc9c84e8f38c34ec31685b62e8b757b0698934d33a64544753

data/.travis.yml

@@ -5,8 +5,10 @@ rvm:
 - ruby-head
 - 2.4.1
 - 2.4.0
+- 2.3.4
 - 2.3.3
-- 2.2
+- 2.2.7
+- 2.2.6
 - 2.1
 - 2.0
 - jruby-head

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## CHANGELOG
 
+### 2.5.0
+
+* Double check UTF-32 only on Ruby versions which contain the bug
+* Highlight unassigned codepoints which are ignorable
+* Bump symbolify dependency
+  * Add special characters (U+FFF9 - U+FFFC)
+  * Non-control separators return ⏎
+* Bump characteristics dependency
+  * Allow GB1988 encoding (7bit ascii-like)
+
 ### 2.4.0
 
 * Extract symbolification logic into extra [symbolify](https://github.com/janlelis/symbolify) gem (includes fixes and non-character detection)

data/README.md

@@ -5,19 +5,20 @@ Ruby library and CLI command that visualizes various Unicode and ASCII/single by
 - Makes analyzing encodings easier
 - Helps you with debugging strings
 - Highlights invalid/special/blank bytes/characters/codepoints
-- Supports *UTF-8*, *UTF-16LE*/*UTF-16BE*, *UTF-32LE*/*UTF-32BE*, *ISO-8859-X*, *Windows-125X*, *IBMX*, *CP85X*, *macX*, *TIS-620*/*Windows-874*, *KOI8-R*/*KOI8-U*, 7-Bit *ASCII*, and arbitrary *BINARY* data
+- Supports *UTF-8*, *UTF-16LE*/*UTF-16BE*, *UTF-32LE*/*UTF-32BE*, *ISO-8859-X*, *Windows-125X*, *IBMX*, *CP85X*, *macX*, *TIS-620*/*Windows-874*, *KOI8-R*/*KOI8-U*, 7-Bit *ASCII*/*GB1988*, and arbitrary *BINARY* data
 
 ## Color Coding
 
 Each byte of the given string is highlighted using the following mechanism (characters -> codepoints):
 
 - Red for invalid bytes
-- Orange for unassigned bytes/characters
-- Blue for control characters
 - Light blue for blanks
+- Blue for control characters
 - Non-control formatting characters in pink
 - Green for marks (Unicode only)
-- Random color for all other characters
+- Orange for unassigned codepoints
+- Lighter orange for unassigned codepoints which are also ignorable
+- Random color for all other codepoints
 
 ## Setup
 
@@ -113,6 +114,7 @@ Example in Ruby: `unibits "🌫 Idiosyncrätic ℜսᖯʏ", encoding: 'ascii'
 Also see
 
 - [Ruby's Encoding class](https://ruby-doc.org/core/Encoding.html)
+- [Symbolify gem](https://github.com/janlelis/symbolify)
 - [Characteristics gem](https://github.com/janlelis/characteristics)
 - [UTF-8 (Wikipedia)](https://en.wikipedia.org/wiki/UTF-8#Description)
 - [UTF-16 (Wikipedia)](https://en.wikipedia.org/wiki/UTF-16#Description)

data/bin/unibits

@@ -53,11 +53,12 @@ if argv[:help]
   #{Paint["COLOR CODING", :underline]}
 
   #{Paint["invalid",    Unibits::COLORS[:invalid]]}
-  #{Paint["unassigned", Unibits::COLORS[:unassigned]]}
-  #{Paint["control",    Unibits::COLORS[:control]]}
   #{Paint["blank",      Unibits::COLORS[:blank]]}
+  #{Paint["control",    Unibits::COLORS[:control]]}
   #{Paint["format",     Unibits::COLORS[:format]]}
   #{Paint["mark",       Unibits::COLORS[:mark]]}
+  #{Paint["unassigned", Unibits::COLORS[:unassigned]]}
+  #{Paint["unassigned and ignorable", Unibits::COLORS[:ignorable]]}
 
   #{Paint["STATS", :underline]}
 

data/lib/unibits.rb

@@ -22,16 +22,18 @@ module Unibits
       /^TIS-620$/,
       /^Windows-874$/,
       /^KOI8/,
+      /^GB1988$/,
     )
   ).sort.freeze
 
   COLORS = {
     invalid: "#FF0000",
-    unassigned: "#FF5500",
     control: "#0000FF",
     blank: "#33AADD",
     format: "#FF00FF",
     mark: "#228822",
+    unassigned: "#FF5500",
+    ignorable: "#FFAA00",
   }
 
   DEFAULT_TERMINAL_WIDTH = 80
@@ -81,7 +83,17 @@ module Unibits
     puts
     string.each_char{ |char|
       char_info = Characteristics.create_for_type(char, type)
-      double_check_utf32_validness!(char, char_info)
+
+      if  RUBY_VERSION >= "2.4.1" ||
+          RUBY_VERSION < "2.4.0" && RUBY_VERSION >= "2.3.4" ||
+          RUBY_VERSION < "2.3.0" && RUBY_VERSION >= "2.2.7" ||
+          char_info.encoding.name[0, 6] != "UTF-32" ||
+          !char_info.valid?
+        # bug is fixed or not relevant
+      else
+        double_check_utf32_validness!(char, char_info)
+      end
+
       current_color = determine_char_color(char_info)
 
       current_encoding_error = nil if char_info.valid?
@@ -239,7 +251,11 @@ module Unibits
     if !char_info.valid?
       COLORS[:invalid]
     elsif !char_info.assigned?
-      COLORS[:unassigned]
+      if char_info.unicode? && char_info.ignorable?
+        COLORS[:ignorable]
+      else
+        COLORS[:unassigned]
+      end
     elsif char_info.blank?
       COLORS[:blank]
     elsif char_info.control?
@@ -315,7 +331,6 @@ module Unibits
   end
 
   def self.double_check_utf32_validness!(char, char_info)
-    return if RUBY_VERSION > "2.4.0" || char_info.encoding.name[0, 6] != "UTF-32" || !char_info.valid?
     byte_values = char.b.unpack("C*")
     le = char_info.encoding.name == 'UTF-32LE'
     if  byte_values[le ? 2 : 1] > 16 ||

data/lib/unibits/version.rb

@@ -1,4 +1,4 @@
 module Unibits
-  VERSION = "2.4.0".freeze
+  VERSION = "2.5.0".freeze
   UNICODE_VERSION = "9.0.0".freeze
 end

data/spec/unibits_spec.rb

@@ -67,6 +67,13 @@ describe Unibits do
       result.must_match "01000011"
     end
 
+    it "works with GB1988" do
+      result = Paint.unpaint(Unibits.visualize("ASCII string".force_encoding('GB1988')))
+      result.must_match "C"
+      result.must_match "43"
+      result.must_match "01000011"
+    end
+
     it "works with 'ISO-8859-X' encodings" do
       string = "\xBC Idiosyncr\xE4tic\n\x91".force_encoding("ISO-8859-1")
       result = Paint.unpaint(Unibits.visualize(string))

data/unibits.gemspec

@@ -19,8 +19,8 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency 'paint', '>= 0.9', '< 3.0'
   gem.add_dependency 'unicode-display_width', '~> 1.1'
-  gem.add_dependency 'symbolify', '~> 1.0'
-  gem.add_dependency 'characteristics', '>= 0.5.2'
+  gem.add_dependency 'symbolify', '~> 1.2'
+  gem.add_dependency 'characteristics', '>= 0.7'
   gem.add_dependency 'rationalist', '~> 2.0'
 
   gem.required_ruby_version = "~> 2.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unibits
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.5.0
 platform: ruby
 authors:
 - Jan Lelis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-25 00:00:00.000000000 Z
+date: 2017-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: paint
@@ -50,28 +50,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: characteristics
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.2
+        version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.2
+        version: '0.7'
 - !ruby/object:Gem::Dependency
   name: rationalist
   requirement: !ruby/object:Gem::Requirement