unibits 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d5f49745b4dd8f80a9b4c87c3810b4ddc5dbe66
-  data.tar.gz: a00bc08a018d1c1e40d9560f48cf64e53aa4e20b
+  metadata.gz: 93d43c363159fed623abd9acd021e0aa6ed90061
+  data.tar.gz: 7fe58c18ca55c1a595345235fa3cf6fa9b0d3635
 SHA512:
-  metadata.gz: 865101498c37b0d480846c6eb9a613942becef9f0f0fa02d01ed78ca1dc8bd4d7d3ae10ce3856ab76ded888278e27da6962657de65ea96c464920c5e3c91866d
-  data.tar.gz: 39362d02cc5f8b8f0c6df3cc3b7f1bb597eef0eb8e2fb828b16f95de8e6a8e8b41faa13d03a33fcb1d2890b7625a855a75924351dca76832994cf50d1af95e9f
+  metadata.gz: 4b580cbfc775274bd9790e0b000a039ed304ffe4e5f41349f5e445118154b9351d5084dfe204c77203b0f4dd4b3e777fa3119e3b47bbba0a422df89bdfec6bad
+  data.tar.gz: 54da485c62dcf7390dde62460d977472130e1c3c6d091e3fa3a71be07c3854260c06c2eb067f7ac2abc96d934e69c6393861c06197a519d1b432f541bc9cf0f8

data/CHANGELOG.md

@@ -1,10 +1,16 @@
 ## CHANGELOG
 
+### 2.1.0
+
+* Support more encoding: IBMX, CP85X, macX, TIS-620/Windows-874, and KOI8-X
+* Highlight non-control formatting characters in pink
+* Improve `unibits --help` command
+
 ### 2.0.0
 
 * Support more encodings: ISO-8859-X and Windows-125X
 * Add three HANGUL characters (U+115F, U+1160, U+3164) to list of possible white spaces
-* Move character handling to separate gem. It is called "characteristics".
+* Move character handling to separate gem. It is called [characteristics](https://github.com/janlelis/characteristics).
 * Highlight control chars in blue and blanks in light blue
 * Handle encodings that are not convertible to UTF-8
 

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    unibits (2.0.0)
-      characteristics (~> 0.2.0)
+    unibits (2.1.0)
+      characteristics (~> 0.3.0)
       paint (>= 0.9, < 3.0)
       rationalist (~> 2.0)
       unicode-display_width (~> 1.1)
@@ -17,7 +17,7 @@ GEM
     boson-more (0.2.2)
       boson (>= 1.2.0)
     cd (1.0.1)
-    characteristics (0.2.0)
+    characteristics (0.3.0)
       unicode-categories (~> 1.1, >= 1.1.2)
     clipboard (1.0.6)
     coderay (1.1.1)

data/README.md

@@ -1,20 +1,21 @@
 # unibits | Reveal the Unicode [![[version]](https://badge.fury.io/rb/unibits.svg)](http://badge.fury.io/rb/unibits)  [![[travis]](https://travis-ci.org/janlelis/unibits.svg)](https://travis-ci.org/janlelis/unibits)
 
-Ruby library and CLI command that visualizes various Unicode and ASCII encodings in the terminal:
+Ruby library and CLI command that visualizes various Unicode and ASCII/single byte encodings in the terminal:
 
 - Makes analyzing encodings easier
 - Helps you with debugging strings
-- Supports **UTF-8**, **UTF-16LE**/**UTF-16BE**, **UTF-32LE**/**UTF-32BE**, **ISO-8859-X**, **Windows-125X** arbitrary **BINARY** data, and **ASCII**
-- Highlights invalid encodings
+- Supports **UTF-8**, **UTF-16LE**/**UTF-16BE**, **UTF-32LE**/**UTF-32BE**, **ISO-8859-X**, **Windows-125X**, **IBMX**, **CP85X**, **macX**, **TIS-620**/**Windows-874**, **KOI8-R**/**KOI8-U**, arbitrary **BINARY** data, and 7-Bit **ASCII**
+- Highlights invalid/special/blank bytes/characters/codepoints
 
 ## Color Coding
 
-Each byte of the given string is highlighted using the following mechanism:
+Each byte of the given string is highlighted using the following mechanism (characters -> codepoints):
 
 - Red for invalid bytes
 - Orange for unassigned bytes/characters
 - Blue for control characters
 - Light blue for blanks
+- Non-control formatting characters in pink
 - Random color for all other characters
 
 ## Setup

data/bin/unibits

@@ -33,7 +33,7 @@ if argv[:help]
 
   #{Paint["DESCRIPTION", :underline]}
 
-  Visualizes ASCII and variouus Unicode encodings in the terminal.
+  Visualizes Unicode and ASCII/single byte encodings in the terminal.
 
   #{Paint["USAGE", :underline]}
 
@@ -48,7 +48,14 @@ if argv[:help]
 
   #{Paint["ENCODINGS", :underline]}
 
-  #{Unibits::SUPPORTED_ENCODINGS.join(', ')}
+  #{Unibits::SUPPORTED_ENCODINGS.join(', ').scan(/.{,80}(?> |\z)/).join("\n  ")}
+  #{Paint["COLOR CODING", :underline]}
+
+  #{Paint["invalid",    Unibits::COLORS[:invalid]]}
+  #{Paint["unassigned", Unibits::COLORS[:unassigned]]}
+  #{Paint["control",    Unibits::COLORS[:control]]}
+  #{Paint["blank",      Unibits::COLORS[:blank]]}
+  #{Paint["format",     Unibits::COLORS[:format]]}
 
   #{Paint["STATS", :underline]}
 

data/lib/unibits/symbolify.rb

@@ -1,6 +1,6 @@
 module Unibits
   module Symbolify
-    NO_UTF8_CONVERTER = /^Windows-1258/
+    NO_UTF8_CONVERTER = /^(Windows-1258|IBM864|macCentEuro|macThai)/
     ASCII_CHARS = "\x20-\x7E".freeze
     ASCII_CONTROL_CODEPOINTS = "\x00-\x1F\x7F".freeze
     ASCII_CONTROL_SYMBOLS = "\u{2400}-\u{241F}\u{2421}".freeze
@@ -354,6 +354,27 @@ module Unibits
       0xE01EF => "VS256",
     }.freeze
 
+    INTERESTING_BYTES_ENCODINGS = {
+      0xD8 => /^macCroatian/,
+      0xF0 => /^mac(Iceland|Roman|Turkish)/,
+      0xFD => /^(ISO-8859-8|Windows-(1255|1256))/,
+      0xFE => /^(ISO-8859-8|Windows-(1255|1256))/,
+    }.freeze
+
+    INTERESTING_BYTES_VALUES = {
+      0xD8 => "Logo",
+      0xF0 => "Logo",
+      0xFD => "LRM",
+      0xFE => "RLM",
+    }.freeze
+
+    MAC_KEY_SYMBOLS = {
+      0x11 => "⌘",
+      0x12 => "⇧",
+      0x13 => "⌥",
+      0x14 => "⌃",
+    }
+
     def self.symbolify(char, char_info)
       if !char_info.valid?
         "�"
@@ -407,13 +428,19 @@ module Unibits
       if char_info.delete?
         char = CONTROL_DELETE_SYMBOL
       elsif char_info.c0?
+        if ord >= 0x11 && ord <= 0x14 && encoding.name =~ /^mac/
+          char = MAC_KEY_SYMBOLS[ord]
+        else
         char = CONTROL_C0_SYMBOLS[ord]
+        end
       elsif char_info.c1?
         char = CONTROL_C1_NAMES[ord]
       elsif no_converter
         treat_char_unconverted = true
       elsif char_info.blank?
         char = "]".encode(encoding) + char + "[".encode(encoding)
+      elsif INTERESTING_BYTES_ENCODINGS[ord] =~ encoding.name
+        char = INTERESTING_BYTES_VALUES[ord]
       end
 
       if no_converter && treat_char_unconverted

data/lib/unibits/version.rb

@@ -1,3 +1,3 @@
 module Unibits
-  VERSION = "2.0.0".freeze
+  VERSION = "2.1.0".freeze
 end

data/lib/unibits.rb

@@ -7,17 +7,31 @@ require "unicode/display_width"
 require "characteristics"
 
 module Unibits
-  SUPPORTED_ENCODINGS = [
-    'UTF-8',
-    'UTF-16LE',
-    'UTF-16BE',
-    'UTF-32LE',
-    'UTF-32BE',
-    'ASCII-8BIT',
-    'US-ASCII',
-    /^ISO-8859-/,
-    /^Windows-125/,
-  ].freeze
+  SUPPORTED_ENCODINGS = Encoding.name_list.grep(
+    Regexp.union(
+      /^UTF-8$/,
+      /^UTF-...E$/,
+      /^ASCII-8BIT$/,
+      /^US-ASCII$/,
+      /^ISO-8859-/,
+      /^Windows-125/,
+      /^IBM/,
+      /^CP85/,
+      /^mac/,
+      /^TIS-620$/,
+      /^Windows-874$/,
+      /^KOI8/,
+    )
+  ).sort.freeze
+
+  COLORS = {
+    invalid: "#FF0000",
+    unassigned: "#FF5500",
+    control: "#0000FF",
+    blank: "#33AADD",
+    format: "#FF00FF",
+  }
+
   DEFAULT_TERMINAL_WIDTH = 80
 
   def self.of(string, encoding: nil, convert: nil, stats: true, wide_ambiguous: false, width: nil)
@@ -69,7 +83,7 @@ module Unibits
 
       current_encoding_error = nil if char_info.valid?
 
-      char.each_byte.with_index{ |byte, index|
+      char.each_byte.with_index{ |byte, byteindex|
         if Paint.unpaint(hex_buffer[-1]).bytesize > cols - 12
           cp_buffer  << "  "
           enc_buffer << "  "
@@ -78,7 +92,7 @@ module Unibits
           separator  << "  "
         end
 
-        if index == 0
+        if byteindex == 0
           if char_info.valid?
             codepoint = "U+%04X" % char.ord
           else
@@ -178,9 +192,7 @@ module Unibits
             end
           end
 
-          cp_buffer[-1] << Paint[
-            codepoint.ljust(10), current_color, :bold
-          ]
+          cp_buffer[-1] << Paint[ codepoint.ljust(10), current_color, :bold ]
 
           symbolified_char = Symbolify.symbolify(char, char_info)
 
@@ -190,80 +202,16 @@ module Unibits
             padding = 10 - symbolified_char.size
           end
 
-          enc_buffer[-1] << Paint[
-            symbolified_char, current_color
-          ]
+          enc_buffer[-1] << Paint[ symbolified_char, current_color ]
           enc_buffer[-1] << " " * padding if padding > 0
         else
           cp_buffer[-1]  << " " * 10
           enc_buffer[-1] << " " * 10
         end
 
-        hex_buffer[-1] << Paint[
-          ("%02X" % byte).ljust(10, " "), current_color
-        ]
-
-        bin_byte_complete = byte.to_s(2).rjust(8, "0")
-
-        if !char_info.valid?
-          bin_byte_1 = bin_byte_complete
-          bin_byte_2 = ""
-        else
-          case encoding_name
-          when 'US-ASCII'
-            bin_byte_1 = bin_byte_complete[0...1]
-            bin_byte_2 = bin_byte_complete[1...8]
-          when 'ASCII-8BIT'
-            bin_byte_1 = ""
-            bin_byte_2 = bin_byte_complete
-          when 'UTF-8'
-            if index == 0
-              if bin_byte_complete =~ /^(0|1{2,4}0)([01]+)$/
-                bin_byte_1 = $1
-                bin_byte_2 = $2
-              else
-                bin_byte_1 = ""
-                bin_byte_2 = bin_byte_complete
-              end
-            else
-              bin_byte_1 = bin_byte_complete[0...2]
-              bin_byte_2 = bin_byte_complete[2...8]
-            end
-          when 'UTF-16LE'
-            if char.ord <= 0xFFFF || index == 0 || index == 2
-              bin_byte_1 = ""
-              bin_byte_2 = bin_byte_complete
-            else
-              bin_byte_complete =~ /^(11011[01])([01]+)$/
-              bin_byte_1 = $1
-              bin_byte_2 = $2
-            end
-          when 'UTF-16BE'
-            if char.ord <= 0xFFFF || index == 1 || index == 3
-              bin_byte_1 = ""
-              bin_byte_2 = bin_byte_complete
-            else
-              bin_byte_complete =~ /^(11011[01])([01]+)$/
-              bin_byte_1 = $1
-              bin_byte_2 = $2
-            end
-          when 'UTF-32LE', 'UTF-32BE'
-            bin_byte_1 = ""
-            bin_byte_2 = bin_byte_complete
-          when /^(ISO-8859-|Windows-125)/
-            bin_byte_1 = ""
-            bin_byte_2 = bin_byte_complete
-          end
-        end
-
-        bin_buffer[-1] << Paint[
-          bin_byte_1, current_color
-        ] unless !bin_byte_1 || bin_byte_1.empty?
-
-        bin_buffer[-1] << Paint[
-          bin_byte_2, current_color, :underline
-        ] unless !bin_byte_2 || bin_byte_2.empty?
+        hex_buffer[-1] << Paint[ ("%02X" % byte).ljust(10, " "), current_color ]
 
+        bin_buffer[-1] << highlight_bits(byte, char, char_info, current_color, byteindex)
         bin_buffer[-1] << "  "
       }
     }
@@ -283,13 +231,15 @@ module Unibits
 
   def self.determine_char_color(char_info)
     if !char_info.valid?
-      "#FF0000"
+      COLORS[:invalid]
     elsif !char_info.assigned?
-      "#FF5500"
+      COLORS[:unassigned]
     elsif char_info.control?
-      "#0000FF"
+      COLORS[:control]
     elsif char_info.blank?
-      "#33AADD"
+      COLORS[:blank]
+    elsif char_info.format?
+      COLORS[:format]
     else
       random_color
     end
@@ -298,4 +248,61 @@ module Unibits
   def self.random_color
     "%.2x%.2x%.2x" % [rand(90) + 60, rand(90) + 60, rand(90) + 60]
   end
+
+  def self.highlight_bits(byte, char, char_info, current_color, byteindex)
+    bin_byte_complete = byte.to_s(2).rjust(8, "0")
+
+    if !char_info.valid?
+      bin_byte_1 = bin_byte_complete
+      bin_byte_2 = ""
+    else
+      case char_info.encoding.name
+      when 'US-ASCII'
+        bin_byte_1 = bin_byte_complete[0...1]
+        bin_byte_2 = bin_byte_complete[1...8]
+      when 'ASCII-8BIT'
+        bin_byte_1 = ""
+        bin_byte_2 = bin_byte_complete
+      when 'UTF-8'
+        if byteindex == 0
+          if bin_byte_complete =~ /^(0|1{2,4}0)([01]+)$/
+            bin_byte_1 = $1
+            bin_byte_2 = $2
+          else
+            bin_byte_1 = ""
+            bin_byte_2 = bin_byte_complete
+          end
+        else
+          bin_byte_1 = bin_byte_complete[0...2]
+          bin_byte_2 = bin_byte_complete[2...8]
+        end
+      when 'UTF-16LE'
+        if char.ord <= 0xFFFF || byteindex == 0 || byteindex == 2
+          bin_byte_1 = ""
+          bin_byte_2 = bin_byte_complete
+        else
+          bin_byte_complete =~ /^(11011[01])([01]+)$/
+          bin_byte_1 = $1
+          bin_byte_2 = $2
+        end
+      when 'UTF-16BE'
+        if char.ord <= 0xFFFF || byteindex == 1 || byteindex == 3
+          bin_byte_1 = ""
+          bin_byte_2 = bin_byte_complete
+        else
+          bin_byte_complete =~ /^(11011[01])([01]+)$/
+          bin_byte_1 = $1
+          bin_byte_2 = $2
+        end
+      else
+        bin_byte_1 = ""
+        bin_byte_2 = bin_byte_complete
+      end
+    end
+
+    res = ""
+    res << Paint[ bin_byte_1, current_color ]             unless !bin_byte_1 || bin_byte_1.empty?
+    res << Paint[ bin_byte_2, current_color, :underline ] unless !bin_byte_2 || bin_byte_2.empty?
+    res
+  end
 end

data/spec/unibits_spec.rb

@@ -67,7 +67,7 @@ describe Unibits do
       result.must_match "01000011"
     end
 
-    it "works with 'ISO-8859-' encodings" do
+    it "works with 'ISO-8859-X' encodings" do
       string = "\xBC Idiosyncr\xE4tic\n\x91".force_encoding("ISO-8859-1")
       result = Paint.unpaint(Unibits.visualize(string))
       result.must_match "BC"  # ¼
@@ -76,7 +76,7 @@ describe Unibits do
       result.must_match "PU1" # C1 name for \x91
     end
 
-    it "works with 'Windows-125' encodings" do
+    it "works with 'Windows-125X' encodings" do
       string = "\xBC Idiosyncr\xE4tic\n\x81".force_encoding("Windows-1252")
       result = Paint.unpaint(Unibits.visualize(string))
       result.must_match "BC"  # ¼
@@ -85,6 +85,46 @@ describe Unibits do
       result.must_match "n/a" # \x81 is not assigned
     end
 
+    it "works with 'IBMX' encodings" do
+      string = "\xFE Idiosyncr\x84tic\n".force_encoding("IBM437")
+      result = Paint.unpaint(Unibits.visualize(string))
+      result.must_match "FE"  # ■
+      result.must_match "84"  # ä
+      result.must_match "␊"   # \n
+    end
+
+    it "works with 'CP85X' encodings" do
+      string = "\xFE Idiosyncr\x84tic\n".force_encoding("CP850")
+      result = Paint.unpaint(Unibits.visualize(string))
+      result.must_match "FE"  # ■
+      result.must_match "84"  # ä
+      result.must_match "␊"   # \n
+    end
+
+    it "works with 'macX' encodings" do
+      string = "\xBD Idiosyncr\x8Atic \x11 \xF0\n".force_encoding("macRoman")
+      result = Paint.unpaint(Unibits.visualize(string))
+      result.must_match "BD"   # Ω
+      result.must_match "8A"   # ä
+      result.must_match "Logo" # \xF0
+      result.must_match "⌘"    # \x11
+      result.must_match "␊"    # \n
+    end
+
+    it "works with 'TIS-620/Windows-874' encodings" do
+      string = "\xA4 Idiosyncratic\n".force_encoding("TIS-620")
+      result = Paint.unpaint(Unibits.visualize(string))
+      result.must_match "A4"  # ค
+      result.must_match "␊"   # \n
+    end
+
+    it "works with 'KOI8-X' encodings" do
+      string = "\xE9\xE4\xE9\xEF\xF3\xF9\xEE\xE3\xF2\xE1\xF4\xE9\xE3\n".force_encoding("KOI8-R")
+      result = Paint.unpaint(Unibits.visualize(string))
+      result.must_match "F9"  # Ы
+      result.must_match "␊"   # \n
+    end
+
     describe "invalid UTF-8 encodings" do
       it "- unexpected continuation byte (1/2)" do
         string = "abc\x80efg"
@@ -218,6 +258,22 @@ describe Unibits do
         result.must_match "incompl."
         result.must_match "�"
       end
+
+      # TODO implement when https://bugs.ruby-lang.org/issues/13292 is released
+
+      # it "- too large codepoint" do
+      #   string = "\x00\x00\x11\x00".force_encoding("UTF-32LE")
+      #   result = Paint.unpaint(Unibits.visualize(string))
+      #   result.must_match "�"
+      #   result.must_match /toolarge.*toolarge.*toolarge.*toolarge/m
+      # end
+
+      # it "- has surrogate" do
+      #   string = "\x00\xD8\x00\x00".force_encoding("UTF-32LE")
+      #   result = Paint.unpaint(Unibits.visualize(string))
+      #   result.must_match "�"
+      #   result.must_match "sur.gate"
+      # end
     end
 
     describe "invalid ASCII encodings" do

data/unibits.gemspec

@@ -12,14 +12,14 @@ Gem::Specification.new do |gem|
   gem.homepage      = "https://github.com/janlelis/unibits"
   gem.license       = "MIT"
 
-  gem.files         = Dir["{**/}{.*,*}"].select{ |path| File.file?(path) && path !~ /^pkg/ }
+  gem.files         = Dir["{**/}{.*,*}"].select{ |path| File.file?(path) && path !~ /^(pkg|screenshots)/ }
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
   gem.add_dependency 'paint', '>= 0.9', '< 3.0'
   gem.add_dependency 'unicode-display_width', '~> 1.1'
-  gem.add_dependency 'characteristics', '~> 0.2.0'
+  gem.add_dependency 'characteristics', '~> 0.3.0'
   gem.add_dependency 'rationalist', '~> 2.0'
 
   gem.required_ruby_version = "~> 2.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unibits
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Jan Lelis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-13 00:00:00.000000000 Z
+date: 2017-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: paint
@@ -50,14 +50,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
   name: rationalist
   requirement: !ruby/object:Gem::Requirement
@@ -96,15 +96,6 @@ files:
 - lib/unibits/kernel_method.rb
 - lib/unibits/symbolify.rb
 - lib/unibits/version.rb
-- screenshots/ascii.invalid.png
-- screenshots/ascii.png
-- screenshots/binary.png
-- screenshots/utf-16be.png
-- screenshots/utf-16le.png
-- screenshots/utf-32be.png
-- screenshots/utf-32le.png
-- screenshots/utf-8.invalid.png
-- screenshots/utf-8.png
 - spec/unibits_spec.rb
 - unibits.gemspec
 homepage: https://github.com/janlelis/unibits