unibits 2.4.0 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e9d89e598970e15c646b6d3dce0969afe24204fe
-  data.tar.gz: '0473794df2d445d3d0bc5d6f5c697d0c148de119'
+SHA256:
+  metadata.gz: 129aed0dcdc3467e6acf759da5068f140e352d52f44a23b2fa304db2227ebb88
+  data.tar.gz: f64d12e177c3c4c88927bd71273bb13855c8eb0d05203a3cbe0381a38604b580
 SHA512:
-  metadata.gz: f9bd47214da917798bb5e40ad7bdaf80d93a0fd2b07d43f825b8819fb81eca54935f4eff5ae3b508a8213895366a2898b049b558e368c6e69b275f7a3a86bb8f
-  data.tar.gz: 71c2583c9935c7ceead69c8d7c694af2f621bc08f952c896d0e24baf8688d747724315e1fe178e98c1e0e2138d830306e3623804ad9dd37b81c100ffa7394e19
+  metadata.gz: 2289fa37584738aa6de0d7239cc1f4cccec419e4db287345ecb440d036a012bfd3055a2ecea4b26b10cacdda50e6965f56516503435789c5a660f90d46394fed
+  data.tar.gz: 93d3e88519d3ec7cfebd37a32e7c5e3d51a23c1273c8708de0f82ce8c0cbe5258be1d49ec3270b5901cd15f94fd3cda288383727d8e772dc1e3849ffbc1f6f00

data/CHANGELOG.md

@@ -1,5 +1,32 @@
 ## CHANGELOG
 
+### 2.9.0
+
+* Unicode 13
+* Improve terminal width detection / Windows support
+
+### 2.8.0
+
+* Unicode 12
+
+### 2.7.0
+
+* Unicode 11
+
+### 2.6.0
+
+* Support Unicode 10.0
+
+### 2.5.0
+
+* Double check UTF-32 only on Ruby versions which contain the bug
+* Highlight unassigned codepoints which are ignorable
+* Bump symbolify dependency
+  * Add special characters (U+FFF9 - U+FFFC)
+  * Non-control separators return ⏎
+* Bump characteristics dependency
+  * Allow GB1988 encoding (7bit ascii-like)
+
 ### 2.4.0
 
 * Extract symbolification logic into extra [symbolify](https://github.com/janlelis/symbolify) gem (includes fixes and non-character detection)

data/Gemfile

@@ -4,4 +4,4 @@ gemspec
 
 gem 'minitest'
 gem 'rake'
-gem 'irbtools', require: 'irbtools/binding'
+gem 'irbtools', require: 'irbtools/binding' unless RUBY_ENGINE == "jruby"

data/MIT-LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2017 Jan Lelis, mail@janlelis.de
+Copyright (c) 2017-2020 Jan Lelis, https://janlelis.com
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -1,23 +1,26 @@
-# unibits | Reveal the Unicode [![[version]](https://badge.fury.io/rb/unibits.svg)](http://badge.fury.io/rb/unibits)  [![[travis]](https://travis-ci.org/janlelis/unibits.svg)](https://travis-ci.org/janlelis/unibits)
+# unibits | Reveal the Unicode [![[version]](https://badge.fury.io/rb/unibits.svg)](https://badge.fury.io/rb/unibits)  [![[ci]](https://github.com/janlelis/unibits/workflows/Test/badge.svg)](https://github.com/janlelis/unibits/actions?query=workflow%3ATest)
 
 Ruby library and CLI command that visualizes various Unicode and ASCII/single byte encodings in the terminal:
 
 - Makes analyzing encodings easier
 - Helps you with debugging strings
 - Highlights invalid/special/blank bytes/characters/codepoints
-- Supports *UTF-8*, *UTF-16LE*/*UTF-16BE*, *UTF-32LE*/*UTF-32BE*, *ISO-8859-X*, *Windows-125X*, *IBMX*, *CP85X*, *macX*, *TIS-620*/*Windows-874*, *KOI8-R*/*KOI8-U*, 7-Bit *ASCII*, and arbitrary *BINARY* data
+- Supports *UTF-8*, *UTF-16LE*/*UTF-16BE*, *UTF-32LE*/*UTF-32BE*, *ISO-8859-X*, *Windows-125X*, *IBMX*, *CP85X*, *macX*, *TIS-620*/*Windows-874*, *KOI8-R*/*KOI8-U*, 7-Bit *ASCII*/*GB1988*, and arbitrary *BINARY* data
 
 ## Color Coding
 
 Each byte of the given string is highlighted using the following mechanism (characters -> codepoints):
 
 - Red for invalid bytes
-- Orange for unassigned bytes/characters
-- Blue for control characters
 - Light blue for blanks
+- Blue for control characters
 - Non-control formatting characters in pink
 - Green for marks (Unicode only)
-- Random color for all other characters
+- Orange for unassigned codepoints
+- Lighter orange for unassigned codepoints which are also ignorable
+- Random color for all other codepoints
+
+The same colors are used in the higher-level companion tool [uniscribe](https://github.com/janlelis/uniscribe).
 
 ## Setup
 
@@ -110,16 +113,21 @@ Example in Ruby: `unibits "🌫 Idiosyncrätic ℜսᖯʏ", encoding: 'ascii'
 
 ## Notes
 
-Also see
+More info
 
 - [Ruby's Encoding class](https://ruby-doc.org/core/Encoding.html)
-- [Characteristics gem](https://github.com/janlelis/characteristics)
 - [UTF-8 (Wikipedia)](https://en.wikipedia.org/wiki/UTF-8#Description)
 - [UTF-16 (Wikipedia)](https://en.wikipedia.org/wiki/UTF-16#Description)
 - [UTF-32 (Wikipedia)](https://en.wikipedia.org/wiki/UTF-32)
 - [Difference between BINARY and ASCII](http://idiosyncratic-ruby.com/56-us-ascii-8bit.html)
-- [Unicode Micro Libraries for Ruby](https://github.com/janlelis/unicode-x)
+
+Related gems
+
+- [uniscribe](https://github.com/janlelis/uniscribe)
+- [unicopy](https://github.com/janlelis/unicopy)
+- [symbolify](https://github.com/janlelis/symbolify)
+- [characteristics](https://github.com/janlelis/characteristics)
 
 Lots of thanks to @damienklinnert for the motivation and inspiration required to build this! 🎆
 
-Copyright (C) 2017 Jan Lelis <http://janlelis.com>. Released under the MIT license.
+Copyright (C) 2017-2020 Jan Lelis <https://janlelis.com>. Released under the MIT license.

data/Rakefile

@@ -32,7 +32,11 @@ end
 
 desc "#{gemspec.name} | Spec"
 task :spec do
-  sh "for file in spec/*_spec.rb; do ruby $file; done"
+  if RbConfig::CONFIG['host_os'] =~ /mswin|mingw/
+    sh "for %f in (spec/\*.rb) do ruby spec/%f"
+  else
+    sh "for file in spec/*.rb; do ruby $file; done"
+  end
 end
 task default: :spec
 

data/bin/unibits

@@ -44,8 +44,9 @@ if argv[:help]
   --convert <encoding>  | -c | which encoding to convert to (if possible)
   --width <n>           | -w | force a specific number of terminal columns
   --no-stats            |    | no stats header with length info
-  --version             |    | displays version of unibits
   --wide-ambiguous      |    | ambiguous characters
+  --help                |    | this help page
+  --version             |    | displays version of unibits
 
   #{Paint["ENCODINGS", :underline]}
 
@@ -53,11 +54,14 @@ if argv[:help]
   #{Paint["COLOR CODING", :underline]}
 
   #{Paint["invalid",    Unibits::COLORS[:invalid]]}
-  #{Paint["unassigned", Unibits::COLORS[:unassigned]]}
-  #{Paint["control",    Unibits::COLORS[:control]]}
   #{Paint["blank",      Unibits::COLORS[:blank]]}
+  #{Paint["control",    Unibits::COLORS[:control]]}
   #{Paint["format",     Unibits::COLORS[:format]]}
   #{Paint["mark",       Unibits::COLORS[:mark]]}
+  #{Paint["unassigned", Unibits::COLORS[:unassigned]]}
+  #{Paint["unassigned and ignorable", Unibits::COLORS[:ignorable]]}
+
+  random color for other characters
 
   #{Paint["STATS", :underline]}
 

data/lib/unibits.rb

@@ -22,32 +22,38 @@ module Unibits
       /^TIS-620$/,
       /^Windows-874$/,
       /^KOI8/,
+      /^GB1988$/,
     )
   ).sort.freeze
 
   COLORS = {
     invalid: "#FF0000",
-    unassigned: "#FF5500",
     control: "#0000FF",
     blank: "#33AADD",
     format: "#FF00FF",
     mark: "#228822",
+    unassigned: "#FF5500",
+    ignorable: "#FFAA00",
   }
 
   DEFAULT_TERMINAL_WIDTH = 80
 
   def self.of(string, encoding: nil, convert: nil, stats: true, wide_ambiguous: false, width: nil)
-    if !string || string.empty?
-      raise ArgumentError, "no data given to unibits"
-    end
+    string = convert_to_encoding_or_raise(string, encoding, convert)
+
+    puts stats(string, wide_ambiguous: wide_ambiguous) if stats
+    puts visualize(string, wide_ambiguous: wide_ambiguous, width: width)
+  end
+
+  def self.convert_to_encoding_or_raise(string, encoding, convert)
+    raise ArgumentError, "no data given to unibits" if !string || string.empty?
 
     string = string.dup.force_encoding(encoding) if encoding
     string = string.encode(convert) if convert
 
     case string.encoding.name
     when *SUPPORTED_ENCODINGS
-      puts stats(string, wide_ambiguous: wide_ambiguous) if stats
-      puts visualize(string, wide_ambiguous: wide_ambiguous, width: width)
+      string
     when 'UTF-16', 'UTF-32'
       raise ArgumentError, "unibits only supports #{string.encoding.name} with specified endianess, please use #{string.encoding.name}LE or #{string.encoding.name}BE"
     else
@@ -81,7 +87,17 @@ module Unibits
     puts
     string.each_char{ |char|
       char_info = Characteristics.create_for_type(char, type)
-      double_check_utf32_validness!(char, char_info)
+
+      if  RUBY_VERSION >= "2.4.1" ||
+          RUBY_VERSION < "2.4.0" && RUBY_VERSION >= "2.3.4" ||
+          RUBY_VERSION < "2.3.0" && RUBY_VERSION >= "2.2.7" ||
+          char_info.encoding.name[0, 6] != "UTF-32" ||
+          !char_info.valid?
+        # bug is fixed or not relevant
+      else
+        double_check_utf32_validness!(char, char_info)
+      end
+
       current_color = determine_char_color(char_info)
 
       current_encoding_error = nil if char_info.valid?
@@ -230,16 +246,20 @@ module Unibits
   end
 
   def self.determine_terminal_cols
-    STDIN.winsize[1] || DEFAULT_TERMINAL_WIDTH
-  rescue Errno::ENOTTY
-    return DEFAULT_TERMINAL_WIDTH
+    STDIN.winsize[1] || ENV['COLUMNS'] || DEFAULT_TERMINAL_WIDTH
+  rescue Errno::ENOTTY, Errno::EBADF
+    return ENV['COLUMNS'] || DEFAULT_TERMINAL_WIDTH
   end
 
   def self.determine_char_color(char_info)
     if !char_info.valid?
       COLORS[:invalid]
     elsif !char_info.assigned?
-      COLORS[:unassigned]
+      if char_info.unicode? && char_info.ignorable?
+        COLORS[:ignorable]
+      else
+        COLORS[:unassigned]
+      end
     elsif char_info.blank?
       COLORS[:blank]
     elsif char_info.control?
@@ -315,7 +335,6 @@ module Unibits
   end
 
   def self.double_check_utf32_validness!(char, char_info)
-    return if RUBY_VERSION > "2.4.0" || char_info.encoding.name[0, 6] != "UTF-32" || !char_info.valid?
     byte_values = char.b.unpack("C*")
     le = char_info.encoding.name == 'UTF-32LE'
     if  byte_values[le ? 2 : 1] > 16 ||

data/lib/unibits/version.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
+
 module Unibits
-  VERSION = "2.4.0".freeze
-  UNICODE_VERSION = "9.0.0".freeze
+  VERSION = "2.9.0"
+  UNICODE_VERSION = "13.0.0"
 end

data/spec/unibits_spec.rb

@@ -67,6 +67,13 @@ describe Unibits do
       result.must_match "01000011"
     end
 
+    it "works with GB1988" do
+      result = Paint.unpaint(Unibits.visualize("ASCII string".force_encoding('GB1988')))
+      result.must_match "C"
+      result.must_match "43"
+      result.must_match "01000011"
+    end
+
     it "works with 'ISO-8859-X' encodings" do
       string = "\xBC Idiosyncr\xE4tic\n\x91".force_encoding("ISO-8859-1")
       result = Paint.unpaint(Unibits.visualize(string))

data/unibits.gemspec

@@ -5,10 +5,10 @@ require File.dirname(__FILE__) + "/lib/unibits/version"
 Gem::Specification.new do |gem|
   gem.name          = "unibits"
   gem.version       = Unibits::VERSION
-  gem.summary       = "Visualizes encodings."
+  gem.summary       = "Visualizes encodings"
   gem.description   = "Visualizes encodings in the terminal. Supports UTF-8, UTF-16LE, UTF-16BE, UTF-32LE, UTF-32BE, US-ASCII, ASCII-8BIT, and most of Rubies single-byte encodings. Comes as CLI command and as Ruby Kernel method."
   gem.authors       = ["Jan Lelis"]
-  gem.email         = ["mail@janlelis.de"]
+  gem.email         = ["hi@ruby.consulting"]
   gem.homepage      = "https://github.com/janlelis/unibits"
   gem.license       = "MIT"
 
@@ -18,10 +18,10 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_dependency 'paint', '>= 0.9', '< 3.0'
-  gem.add_dependency 'unicode-display_width', '~> 1.1'
-  gem.add_dependency 'symbolify', '~> 1.0'
-  gem.add_dependency 'characteristics', '>= 0.5.2'
-  gem.add_dependency 'rationalist', '~> 2.0'
+  gem.add_dependency 'unicode-display_width', '~> 2.0'
+  gem.add_dependency 'symbolify', '~> 1.4'
+  gem.add_dependency 'characteristics', '~> 1.3'
+  gem.add_dependency 'rationalist', '~> 2.0', '>= 2.0.1'
 
-  gem.required_ruby_version = "~> 2.0"
+  gem.required_ruby_version = ">= 2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unibits
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.9.0
 platform: ruby
 authors:
 - Jan Lelis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-25 00:00:00.000000000 Z
+date: 2020-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: paint
@@ -36,42 +36,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: symbolify
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: characteristics
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.2
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.2
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rationalist
   requirement: !ruby/object:Gem::Requirement
@@ -79,6 +79,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -86,18 +89,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.1
 description: Visualizes encodings in the terminal. Supports UTF-8, UTF-16LE, UTF-16BE,
   UTF-32LE, UTF-32BE, US-ASCII, ASCII-8BIT, and most of Rubies single-byte encodings.
   Comes as CLI command and as Ruby Kernel method.
 email:
-- mail@janlelis.de
+- hi@ruby.consulting
 executables:
 - unibits
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -121,7 +126,7 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - "~>"
+  - - ">="
     - !ruby/object:Gem::Version
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
@@ -130,10 +135,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 3.2.3
 signing_key: 
 specification_version: 4
-summary: Visualizes encodings.
+summary: Visualizes encodings
 test_files:
 - spec/unibits_spec.rb

data/.travis.yml

@@ -1,23 +0,0 @@
-sudo: false
-language: ruby
-
-rvm:
-- ruby-head
-- 2.4.1
-- 2.4.0
-- 2.3.3
-- 2.2
-- 2.1
-- 2.0
-- jruby-head
-- jruby-9.1.8.0
-
-cache:
-- bundler
-
-matrix:
-  allow_failures:
-    - rvm: jruby-head
-    - rvm: ruby-head
-    - rvm: 2.0
-#   fast_finish: true