unibits 2.1.0 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93d43c363159fed623abd9acd021e0aa6ed90061
-  data.tar.gz: 7fe58c18ca55c1a595345235fa3cf6fa9b0d3635
+  metadata.gz: 62770bc741ec5a7693759d4e888d46cf7b3860f2
+  data.tar.gz: 171c479f1f4ebdcf11bc97c4e4e0b94bac713abb
 SHA512:
-  metadata.gz: 4b580cbfc775274bd9790e0b000a039ed304ffe4e5f41349f5e445118154b9351d5084dfe204c77203b0f4dd4b3e777fa3119e3b47bbba0a422df89bdfec6bad
-  data.tar.gz: 54da485c62dcf7390dde62460d977472130e1c3c6d091e3fa3a71be07c3854260c06c2eb067f7ac2abc96d934e69c6393861c06197a519d1b432f541bc9cf0f8
+  metadata.gz: 793cdfeca34183b1b1f6cc219ddd822b9faac5542e92f1ec1065c3fd6ff07c38762becc5312d9bd29fbe3f976bd6eabb3e4c5e228bcfbe8e6340e90b5407234e
+  data.tar.gz: 9691e9072ff3084706e5dfe1ed51b1e6c206d2b11a9eaf710a37ee7b5f1e09b15a6eb3d85e3cb4517a64ef9e5b6aa0bb6bfadee72285487a4a1d0f3f66fa32bd

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## CHANGELOG
 
+### 2.1.1
+
+* Proper UTF-32 validness / invalid codepoint highlighting, see https://bugs.ruby-lang.org/issues/13292
+
 ### 2.1.0
 
 * Support more encoding: IBMX, CP85X, macX, TIS-620/Windows-874, and KOI8-X

data/README.md

@@ -4,8 +4,8 @@ Ruby library and CLI command that visualizes various Unicode and ASCII/single by
 
 - Makes analyzing encodings easier
 - Helps you with debugging strings
-- Supports **UTF-8**, **UTF-16LE**/**UTF-16BE**, **UTF-32LE**/**UTF-32BE**, **ISO-8859-X**, **Windows-125X**, **IBMX**, **CP85X**, **macX**, **TIS-620**/**Windows-874**, **KOI8-R**/**KOI8-U**, arbitrary **BINARY** data, and 7-Bit **ASCII**
 - Highlights invalid/special/blank bytes/characters/codepoints
+- Supports *UTF-8*, *UTF-16LE*/*UTF-16BE*, *UTF-32LE*/*UTF-32BE*, *ISO-8859-X*, *Windows-125X*, *IBMX*, *CP85X*, *macX*, *TIS-620*/*Windows-874*, *KOI8-R*/*KOI8-U*, 7-Bit *ASCII*, and arbitrary *BINARY* data
 
 ## Color Coding
 
@@ -53,7 +53,7 @@ unibits "🌫 Idiosyncrätic ℜսᖯʏ"
 - *wide-ambiguous*: Treat characters of ambiguous width as 2 spaces instead of 1 ([more info](https://github.com/janlelis/unicode-display_width))
 - *width (w)*: Set a custom column width, if not set, *unibits* will retrieve it from the terminal or just use 80
 
-## Output of Different Valid Encodings
+## Examples of Valid Encodings
 ### UTF-8
 
 CLI: `$ unibits -e utf-8 -c utf-8 "🌫 Idiosyncrätic ℜսᖯʏ"`
@@ -70,22 +70,6 @@ Ruby: `unibits "🌫 Idiosyncrätic ℜսᖯʏ", encoding: 'utf-8', convert:
 
 ![Screenshot UTF-16LE](/screenshots/utf-16le.png?raw=true "UTF-16LE")
 
-### UTF-16BE
-
-CLI: `$ unibits -e utf-8 -c utf-16be "🌫 Idiosyncrätic ℜսᖯʏ"`
-
-Ruby: `unibits "🌫 Idiosyncrätic ℜսᖯʏ", encoding: 'utf-8', convert: 'utf-16be'`
-
-![Screenshot UTF-16BE](/screenshots/utf-16be.png?raw=true "UTF-16BE")
-
-### UTF-32LE
-
-CLI: `$ unibits -e utf-8 -c utf-32le "🌫 Idiosyncrätic ℜսᖯʏ"`
-
-Ruby: `unibits "🌫 Idiosyncrätic ℜսᖯʏ", encoding: 'utf-8', convert: 'utf-32le'`
-
-![Screenshot UTF-32LE](/screenshots/utf-32le.png?raw=true "UTF-32LE")
-
 ### UTF-32BE
 
 CLI: `$ unibits -e utf-8 -c utf-32be "🌫 Idiosyncrätic ℜսᖯʏ"`
@@ -106,11 +90,11 @@ Ruby: `unibits "🌫 Idiosyncrätic ℜսᖯʏ", encoding: 'binary'`
 
 CLI: `$ unibits -e utf-8 -c ascii "ascii"`
 
-Ruby: `unibits "ASCII String", encoding: 'utf-8', convert: 'ascii'`
+Ruby: `unibits "ascii", encoding: 'utf-8', convert: 'ascii'`
 
 ![Screenshot ASCII](/screenshots/ascii.png?raw=true "ASCII")
 
-## Invalid Encodings
+## Examples of Invalid Encodings
 ### UTF-8
 
 Example in Ruby: `unibits "unexpected \x80 | not enough \xF0\x9F\x8C | overlong \xE0\x81\x81 | surrogate \xED\xA0\x80 | too large \xF5\x8F\xBF\xBF"`
@@ -123,10 +107,6 @@ Example in Ruby: `unibits "🌫 Idiosyncrätic ℜսᖯʏ", encoding: 'ascii'
 
 ![Screenshot invalid ASCII](/screenshots/ascii.invalid.png?raw=true "Invalid ASCII")
 
-### BINARY
-
-Not possible to produce invalid binary strings
-
 ## Notes
 
 Also see

data/lib/unibits.rb

@@ -79,6 +79,7 @@ module Unibits
     puts
     string.each_char{ |char|
       char_info = Characteristics.create_for_type(char, type)
+      double_check_utf32_validness!(char, char_info)
       current_color = determine_char_color(char_info)
 
       current_encoding_error = nil if char_info.valid?
@@ -184,10 +185,13 @@ module Unibits
                 codepoint = "invalid"
               end
             when 'UTF-32LE', 'UTF-32BE'
-              if char.bytesize != "4"
+              if char.bytesize % 4 != 0
                 codepoint = "incompl."
+              elsif char.b.unpack("C*")[encoding_name == 'UTF-32LE' ? 2 : 1] > 16 ||
+                    char.b.unpack("C*")[encoding_name == 'UTF-32LE' ? 3 : 0] > 0
+                codepoint = "toolarge"
               else
-                codepoint = "invalid"
+                codepoint = "sur.gate"
               end
             end
           end
@@ -305,4 +309,15 @@ module Unibits
     res << Paint[ bin_byte_2, current_color, :underline ] unless !bin_byte_2 || bin_byte_2.empty?
     res
   end
+
+  def self.double_check_utf32_validness!(char, char_info)
+    return if RUBY_VERSION > "2.4.0" || char_info.encoding.name[0, 6] != "UTF-32" || !char_info.valid?
+    byte_values = char.b.unpack("C*")
+    le = char_info.encoding.name == 'UTF-32LE'
+    if  byte_values[le ? 2 : 1] > 16 ||
+        byte_values[le ? 3 : 0] > 0 ||
+        byte_values[le ? 1 : 2] >= 216  && byte_values[le ? 1 : 2] <= 223
+      char_info.instance_variable_set(:@is_valid, false)
+    end
+  end
 end

data/lib/unibits/version.rb

@@ -1,3 +1,3 @@
 module Unibits
-  VERSION = "2.1.0".freeze
+  VERSION = "2.1.1".freeze
 end

data/spec/unibits_spec.rb

@@ -259,21 +259,26 @@ describe Unibits do
         result.must_match "�"
       end
 
-      # TODO implement when https://bugs.ruby-lang.org/issues/13292 is released
-
-      # it "- too large codepoint" do
-      #   string = "\x00\x00\x11\x00".force_encoding("UTF-32LE")
-      #   result = Paint.unpaint(Unibits.visualize(string))
-      #   result.must_match "�"
-      #   result.must_match /toolarge.*toolarge.*toolarge.*toolarge/m
-      # end
-
-      # it "- has surrogate" do
-      #   string = "\x00\xD8\x00\x00".force_encoding("UTF-32LE")
-      #   result = Paint.unpaint(Unibits.visualize(string))
-      #   result.must_match "�"
-      #   result.must_match "sur.gate"
-      # end
+      it "- too large codepoint (1/2)" do
+        string = "\x00\x00\x11\x00".force_encoding("UTF-32LE")
+        result = Paint.unpaint(Unibits.visualize(string))
+        result.must_match "�"
+        result.must_match "toolarge"
+      end
+
+      it "- too large codepoint (2/2)" do
+        string = "\x00\x00\x00\x01".force_encoding("UTF-32LE")
+        result = Paint.unpaint(Unibits.visualize(string))
+        result.must_match "�"
+        result.must_match "toolarge"
+      end
+
+      it "- has surrogate" do
+        string = "\x00\xD8\x00\x00".force_encoding("UTF-32LE")
+        result = Paint.unpaint(Unibits.visualize(string))
+        result.must_match "�"
+        result.must_match "sur.gate"
+      end
     end
 
     describe "invalid ASCII encodings" do

data/unibits.gemspec

@@ -5,8 +5,8 @@ require File.dirname(__FILE__) + "/lib/unibits/version"
 Gem::Specification.new do |gem|
   gem.name          = "unibits"
   gem.version       = Unibits::VERSION
-  gem.summary       = "Visualizes Unicode encodings."
-  gem.description   = "Visualizes Unicode encodings in the terminal. Supports UTF-8, UTF-16LE, UTF-16BE, UTF-32LE, UTF-32BE, US-ASCII, and ASCII-8BIT. Comes as CLI command and as Ruby Kernel method."
+  gem.summary       = "Visualizes encodings."
+  gem.description   = "Visualizes encodings in the terminal. Supports UTF-8, UTF-16LE, UTF-16BE, UTF-32LE, UTF-32BE, US-ASCII, ASCII-8BIT, and most of Rubies single-byte encodings. Comes as CLI command and as Ruby Kernel method."
   gem.authors       = ["Jan Lelis"]
   gem.email         = ["mail@janlelis.de"]
   gem.homepage      = "https://github.com/janlelis/unibits"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unibits
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.1.1
 platform: ruby
 authors:
 - Jan Lelis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-14 00:00:00.000000000 Z
+date: 2017-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: paint
@@ -72,9 +72,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-description: Visualizes Unicode encodings in the terminal. Supports UTF-8, UTF-16LE,
-  UTF-16BE, UTF-32LE, UTF-32BE, US-ASCII, and ASCII-8BIT. Comes as CLI command and
-  as Ruby Kernel method.
+description: Visualizes encodings in the terminal. Supports UTF-8, UTF-16LE, UTF-16BE,
+  UTF-32LE, UTF-32BE, US-ASCII, ASCII-8BIT, and most of Rubies single-byte encodings.
+  Comes as CLI command and as Ruby Kernel method.
 email:
 - mail@janlelis.de
 executables:
@@ -121,6 +121,6 @@ rubyforge_project:
 rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
-summary: Visualizes Unicode encodings.
+summary: Visualizes encodings.
 test_files:
 - spec/unibits_spec.rb