characteristics 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 382e8a6535fa4152dbdbf7f213a12ba6e5a82830
-  data.tar.gz: c92c1bdf4eb32f8ee66c0dce0d72327f95ca0260
+  metadata.gz: fde954a2c2f2028fb7656a57c18a6556416a2f2f
+  data.tar.gz: 1856bc93124425900c7785c2107965d6620ef194
 SHA512:
-  metadata.gz: e6e886ba4aa8b3605c5a8918e3e97c2982a1843779e780562757373fe30641e7e56e6e8bb0ea90bcca6b20a1ce752780c587a87902f3f346fa42a56c1cca07bd
-  data.tar.gz: 273ee12ec3e4bb2426743ac146790b658bdfc236e396c73d7af369b4cf76d89248fcb074a1515c4280e322a05644feeaf838622782210501fab2c36e7819e384
+  metadata.gz: 7d2e5f27972d6db2c3164ca9a915fd59ed69e2b99f4ec4fb4084a73be0fe8283b5c619c8b3bc6f06547a46592c3ac74bbaf07c2dd1c1afa1bfb5eb98aa6f0719
+  data.tar.gz: 52c27cda5f0e20b7d7b1a7f99c0fc505e2250d6a8263850bb889fb1841ad95984f0f67c3e1d7203c3efac00284fd3d8f92d7fbb36df8881f16586bf4610f9333

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## CHANGELOG
 
+### 0.3.0
+
+* Add soft-hyphen to single byte encodings
+* Add format? property (e.g. RLM)
+* Support more encodings: IBMX, CP85X, macX, TIS-620, Windows-874, KOI8-X
+
 ### 0.2.0
 
 * Fix detection of supported Windows encodings and some unassigned codepoints

data/README.md

@@ -26,6 +26,7 @@ char_info.unicode? # => true / false
 char_info.assigned? # => true / false
 char_info.control? # => true / false
 char_info.blank? # => true / false
+char_info.format? # => true / false
 ```
 
 ## Types of Encodings
@@ -35,7 +36,7 @@ This library knows of four different kinds of encodings:
 - **:unicode** Unicode familiy of multibyte encodings (*UTF-X*)
 - **:ascii** 7-Bit ASCII (*US-ASCII*)
 - **:binary** Arbitrary string (*ASCII-8BIT*)
-- **:byte** Known byte encoding (*ISO-8859-X*, *Windows-125X*)
+- **:byte** Known single byte encoding (*ISO-8859-X*, *Windows-125X*, *IBMX*, *CP85X*, *macX*, *TIS-620*, *Windows-874*, **KOI-X**)
 
 Other encodings are not supported, yet.
 
@@ -51,7 +52,7 @@ Validness is determined by Ruby's String#valid_encoding?
 
 ### `control?`
 
-Control characters are codepoints in the is [C0, delete or C1 control character range](https://en.wikipedia.org/wiki/C0_and_C1_control_codes).
+Control characters are codepoints in the is [C0, delete or C1 control character range](https://en.wikipedia.org/wiki/C0_and_C1_control_codes). Characters in this range of [IBM codepage 437](https://en.wikipedia.org/wiki/Code_page_437) based encodings are always treated as control characters.
 
 ### `assigned?`
 
@@ -63,6 +64,10 @@ Control characters are codepoints in the is [C0, delete or C1 control character
 
 The library includes a list of characters that might not be rendered visually. This list does not include unassigned codepoints, control characters (except for `\t`, `\n`, `\v`, `\f`, `\r`), or special formatting characters (right-to-left marker, variation selectors, etc).
 
+### `format?`
+
+This flag is `true` only for special formatting characters, which are not control characters, like Right-to-left marks. In Unicode, this means codepoints with the General Category of **Cf**.
+
 ## Todo
 
 - Support all non-dummy encodings that Ruby supports

data/lib/characteristics.rb

@@ -14,7 +14,7 @@ class Characteristics
       :binary
     when /^UTF-/
       :unicode
-    when /^ISO-8859-/, /^Windows-125/
+    when /^ISO-8859-/, /^Windows-125/, /^(IBM|CP85)/, /^mac/, 'TIS-620', 'Windows-874', /^KOI8-/
       :byte
     else
       raise ArgumentError, "encoding <#{encoding_name}> not supported"
@@ -41,6 +41,8 @@ class Characteristics
   attr_reader :encoding
 
   def initialize(char)
+    raise ArgumentError, "Do not use abstract Characteristics.new(char) directly, please use Characteristics.create(char)" if self.class == Characteristics
+
     @is_valid = char.valid_encoding?
     @encoding = char.encoding
     @encoding_name = @encoding.name
@@ -51,7 +53,6 @@ class Characteristics
   end
 
   def unicode?
-    false
   end
 
   def assigned?
@@ -62,4 +63,7 @@ class Characteristics
 
   def blank?
   end
+
+  def format?
+  end
 end

data/lib/characteristics/ascii.rb

@@ -16,6 +16,10 @@ class AsciiCharacteristics < Characteristics
     @ord = char.ord if @is_valid
   end
 
+  def unicode?
+    false
+  end
+
   def assigned?
     true
   end
@@ -39,4 +43,8 @@ class AsciiCharacteristics < Characteristics
   def blank?
     @is_valid && ( BLANKS.include?(@ord) || SEPARATORS.include?(@ord) )
   end
+
+  def format?
+    false
+  end
 end

data/lib/characteristics/binary.rb

@@ -21,6 +21,10 @@ class BinaryCharacteristics < Characteristics
     true
   end
 
+  def unicode?
+    false
+  end
+
   def assigned?
     true
   end
@@ -40,4 +44,8 @@ class BinaryCharacteristics < Characteristics
   def blank?
     BLANKS.include?(@ord) || SEPARATORS.include?(@ord)
   end
+
+  def format?
+    false
+  end
 end

data/lib/characteristics/byte.rb

@@ -1,29 +1,42 @@
 class ByteCharacteristics < Characteristics
-  HAS_C1 = /^(ISO-8859-)/
+  HAS_C1 = /^(ISO-8859-|TIS-620)/
 
   UNASSIGNED = {
-    0x81 => /^(Windows-(1250|1252|1253|1254|1255|1257|1258))/,
-    0x83 => /^(Windows-(1250|1257))/,
-    0x88 => /^(Windows-(1250|1253|1257))/,
-    0x8A => /^(Windows-(1253|1255|1257|1258))/,
-    0x8C => /^(Windows-(1253|1255|1257))/,
-    0x8D => /^(Windows-(1252|1253|1254|1255|1258))/,
-    0x8E => /^(Windows-(1253|1254|1255|1258))/,
-    0x8F => /^(Windows-(1252|1253|1254|1255|1258))/,
-
-    0x90 => /^(Windows-(1250|1252|1253|1254|1255|1257|1258))/,
-    0x98 => /^(Windows-(1250|1251|1253|1257))/,
-    0x9A => /^(Windows-(1253|1255|1257|1258))/,
-    0x9C => /^(Windows-(1253|1255|1257))/,
-    0x9D => /^(Windows-(1252|1253|1254|1255|1258))/,
-    0x9E => /^(Windows-(1253|1254|1255|1258))/,
-    0x9F => /^(Windows-(1253|1255|1257))/,
-
+    0x80 => /^(IBM869)/,
+    0x81 => /^(Windows-(1250|1252|1253|1254|1255|1257|1258)|IBM869|Windows-874)/,
+    0x82 => /^(IBM869|Windows-874)/,
+    0x83 => /^(Windows-(1250|1257)|IBM869|Windows-874)/,
+    0x84 => /^(IBM869|Windows-874)/,
+    0x85 => /^(IBM869)/,
+    0x86 => /^(Windows-874)/,
+    0x87 => /^(IBM869|Windows-874)/,
+    0x88 => /^(Windows-(1250|1253|1257)|Windows-874)/,
+    0x89 => /^(Windows-874)/,
+    0x8A => /^(Windows-(1253|1255|1257|1258)|Windows-874)/,
+    0x8B => /^(Windows-874)/,
+    0x8C => /^(Windows-(1253|1255|1257)|Windows-874)/,
+    0x8D => /^(Windows-(1252|1253|1254|1255|1258)|Windows-874)/,
+    0x8E => /^(Windows-(1253|1254|1255|1258)|Windows-874)/,
+    0x8F => /^(Windows-(1252|1253|1254|1255|1258)|Windows-874)/,
+
+    0x90 => /^(Windows-(1250|1252|1253|1254|1255|1257|1258)|macThai|Windows-874)/,
+    0x93 => /^(IBM869)/,
+    0x94 => /^(IBM869)/,
+    0x98 => /^(Windows-(1250|1251|1253|1257)|Windows-874)/,
+    0x99 => /^(Windows-874)/,
+    0x9A => /^(Windows-(1253|1255|1257|1258)|Windows-874)/,
+    0x9B => /^(IBM864|Windows-874)/,
+    0x9C => /^(Windows-(1253|1255|1257)|IBM864|Windows-874)/,
+    0x9D => /^(Windows-(1252|1253|1254|1255|1258)|Windows-874)/,
+    0x9E => /^(Windows-(1253|1254|1255|1258)|Windows-874)/,
+    0x9F => /^(Windows-(1253|1255|1257)|IBM864|macThai|Windows-874)/,
+
+    0xA0 => /^(TIS-620)/,
     0xA1 => /^(ISO-8859-(6|8)|Windows-(1257))/,
     0xA2 => /^(ISO-8859-(6))/,
     0xA3 => /^(ISO-8859-(6))/,
     0xA5 => /^(ISO-8859-(3|6)|Windows-(1257))/,
-    0xA6 => /^(ISO-8859-(6))/,
+    0xA6 => /^(ISO-8859-(6)|IBM864)/,
     0xA7 => /^(ISO-8859-(6))/,
     0xA8 => /^(ISO-8859-(6))/,
     0xA9 => /^(ISO-8859-(6))/,
@@ -70,34 +83,36 @@ class ByteCharacteristics < Characteristics
     0xD2 => /^(ISO-8859-(7|8)|Windows-(1253))/,
     0xD3 => /^(ISO-8859-(8))/,
     0xD4 => /^(ISO-8859-(8))/,
-    0xD5 => /^(ISO-8859-(8))/,
+    0xD5 => /^(ISO-8859-(8)|IBM857)/, # IBM857: Ruby does not support euro sign
     0xD6 => /^(ISO-8859-(8))/,
     0xD7 => /^(ISO-8859-(8))/,
     0xD8 => /^(ISO-8859-(8))/,
     0xD9 => /^(ISO-8859-(8)|Windows-(1255))/,
     0xDA => /^(ISO-8859-(8)|Windows-(1255))/,
-    0xDB => /^(ISO-8859-(6|8|11)|Windows-(1255))/,
-    0xDC => /^(ISO-8859-(6|8|11)|Windows-(1255))/,
-    0xDD => /^(ISO-8859-(6|8|11)|Windows-(1255))/,
-    0xDE => /^(ISO-8859-(6|8|11)|Windows-(1255))/,
+    0xDB => /^(ISO-8859-(6|8|11)|Windows-(1255)|TIS-620|Windows-874)/,
+    0xDC => /^(ISO-8859-(6|8|11)|Windows-(1255)|TIS-620|Windows-874)/,
+    0xDD => /^(ISO-8859-(6|8|11)|Windows-(1255)|TIS-620|Windows-874)/,
+    0xDE => /^(ISO-8859-(6|8|11)|Windows-(1255)|TIS-620|Windows-874)/,
     0xDF => /^(ISO-8859-(6)|Windows-(1255))/,
 
     0xE3 => /^(ISO-8859-(3))/,
+    0xE7 => /^(IBM857)/,
 
-    0xF0 => /^(ISO-8859-(3))/,
+    0xF0 => /^(ISO-8859-(3))/, # mac: Treating F0 as always assigned
+    0xF2 => /^(IBM857)/,
     0xF3 => /^(ISO-8859-(6))/,
     0xF4 => /^(ISO-8859-(6))/,
-    0xF5 => /^(ISO-8859-(6))/,
+    0xF5 => /^(ISO-8859-(6)|macTurkish)/,
     0xF6 => /^(ISO-8859-(6))/,
     0xF7 => /^(ISO-8859-(6))/,
     0xF8 => /^(ISO-8859-(6))/,
     0xF9 => /^(ISO-8859-(6))/,
     0xFA => /^(ISO-8859-(6))/,
     0xFB => /^(ISO-8859-(6|8)|Windows-(1255))/,
-    0xFC => /^(ISO-8859-(6|8|11)|Windows-(1255))/,
-    0xFD => /^(ISO-8859-(6|11))/,
-    0xFE => /^(ISO-8859-(6|11))/,
-    0xFF => /^(ISO-8859-(6|7|8|11)|Windows-(1253|1255))/,
+    0xFC => /^(ISO-8859-(6|8|11)|Windows-(1255)|macThai|TIS-620|Windows-874)/,
+    0xFD => /^(ISO-8859-(6|11)|macThai|TIS-620|Windows-874)/,
+    0xFE => /^(ISO-8859-(6|11)|macThai|TIS-620|Windows-874)/,
+    0xFF => /^(ISO-8859-(6|7|8|11)|Windows-(1253|1255)|IBM864|macGreek|macThai|TIS-620|Windows-874)/, # macGreek: Ruby does not know of soft hyphen at FF
   }.freeze
 
   BLANKS = [
@@ -113,9 +128,22 @@ class ByteCharacteristics < Characteristics
   ].freeze
 
   EXTRA_BLANKS = {
-    0xA0 => /^(ISO-8859-|Windows-125)/,
+    0xA0 => /^(ISO-8859-|Windows-125|macThai|Windows-874)/,
+    0xA1 => /^IBM864/,
+    0xAD => /^(ISO-8859-(?!11)|Windows-125)/,
+    0x9A => /^KOI8-/,
     0x9D => /^Windows-(1256)/,
-    0x9F => /^Windows-(1256)/,
+    0x9E => /^Windows-(1256)/,
+    0xCA => /^mac(?!Thai)/,
+    0xDB => /^macThai/,
+    0xDC => /^macThai/,
+    0xF0 => /^(IBM(?!437|737|86)|IBM869|CP)/,
+    0xFF => /^(IBM(?!864)|CP)/, # |macGreek, but is unnasigned in Ruby
+  }.freeze
+
+  FORMATS = {
+    0xFD => /^(ISO-8859-8|Windows-(1255|1256))/,
+    0xFE => /^(ISO-8859-8|Windows-(1255|1256))/,
   }.freeze
 
   def initialize(char)
@@ -124,12 +152,10 @@ class ByteCharacteristics < Characteristics
   end
 
   def encoding_has_c0?
-    # !!(HAS_C0 =~ @encoding_name)
     true
   end
 
   def encoding_has_delete?
-    # !!(HAS_C0 =~ @encoding_name)
     true
   end
 
@@ -137,6 +163,10 @@ class ByteCharacteristics < Characteristics
     !!(HAS_C1 =~ @encoding_name)
   end
 
+  def unicode?
+    false
+  end
+
   def assigned?
     control? || UNASSIGNED[@ord] !~ @encoding_name
   end
@@ -162,4 +192,8 @@ class ByteCharacteristics < Characteristics
     SEPARATORS.include?(@ord) ||
     EXTRA_BLANKS[@ord] =~ @encoding_name
   end
+
+  def format?
+    FORMATS[@ord] =~ @encoding_name
+  end
 end

data/lib/characteristics/unicode.rb

@@ -100,4 +100,8 @@ class UnicodeCharacteristics < Characteristics
   def blank?
     @is_valid && ( BLANKS.include?(@ord) || SEPARATORS.include?(@ord) )
   end
+
+  def format?
+    @is_valid && @category == "Cf"
+  end
 end

data/lib/characteristics/version.rb

@@ -1,4 +1,4 @@
 class Characteristics
-  VERSION = "0.2.0".freeze
+  VERSION = "0.3.0".freeze
 end
 

data/spec/characteristics_spec.rb

@@ -18,6 +18,10 @@ describe Characteristics do
     Characteristics.create(char.force_encoding(encoding)).blank?
   end
 
+  def format?(char)
+    Characteristics.create(char.force_encoding(encoding)).format?
+  end
+
   describe UnicodeCharacteristics do
     describe "UTF-*" do
       let(:encoding) { "UTF-8" }
@@ -43,6 +47,11 @@ describe Characteristics do
         assert blank? "\x20"
         refute blank? "\x21"
       end
+
+      it "is format or not" do
+        assert format? "\uFFF9"
+        refute format? "\x21"
+      end
     end
   end
 
@@ -68,6 +77,10 @@ describe Characteristics do
         assert blank? "\x20"
         refute blank? "\x21"
       end
+
+      it "is never format" do
+        refute format? "\x21"
+      end
     end
   end
 
@@ -94,6 +107,10 @@ describe Characteristics do
         assert blank? "\x20"
         refute blank? "\x21"
       end
+
+      it "is never format" do
+        refute format? "\x21"
+      end
     end
   end
 
@@ -122,6 +139,10 @@ describe Characteristics do
           assert blank? "\x20"
           refute blank? "\x21"
         end
+
+        it "is never format" do
+          refute format? "\x21"
+        end
       end
 
       # TODO
@@ -142,7 +163,7 @@ describe Characteristics do
       # describe "ISO-8859-16" do
     end
 
-    describe "Windows-*" do
+    describe "Windows-125*" do
       describe "Windows-1252" do
         let(:encoding) { "Windows-1252" }
 
@@ -164,6 +185,10 @@ describe Characteristics do
           assert blank? "\x20"
           refute blank? "\x21"
         end
+
+        it "is never format" do
+          refute format? "\x21"
+        end
       end
 
       # TODO
@@ -177,5 +202,199 @@ describe Characteristics do
       # describe "Windows-1257" do
       # describe "Windows-1258" do
     end
+
+    describe "IBM*, CP85*" do
+      describe "IBM869" do
+        let(:encoding) { "IBM869" }
+
+        it "is always valid" do
+          assert valid? "\x80"
+        end
+
+        it "is assigned or not" do
+          assert assigned? "\x21"
+          refute assigned? "\x80"
+        end
+
+        it "is control or not" do
+          assert control? "\x1E"
+          refute control? "\x67"
+        end
+
+        it "is blank or not" do
+          assert blank? "\x20"
+          refute blank? "\x21"
+        end
+
+        it "is never format" do
+          refute format? "\x21"
+        end
+      end
+
+      # describe "IBM437" do
+      # describe "IBM737" do
+      # describe "IBM775" do
+      # describe "CP850" do
+      # describe "IBM852" do
+      # describe "CP852" do
+      # describe "IBM855" do
+      # describe "CP855" do
+      # describe "IBM857" do
+      # describe "IBM860" do
+      # describe "IBM861" do
+      # describe "IBM862" do
+      # describe "IBM863" do
+      # describe "IBM864" do
+      # describe "IBM865" do
+      # describe "IBM866" do
+    end
+
+    describe "mac*" do
+      describe "macRoman" do
+        let(:encoding) { "macRoman" }
+
+        it "is always valid" do
+          assert valid? "\x80"
+        end
+
+        it "is always assigned" do
+          assert assigned? "\x21"
+        end
+
+        it "is control or not" do
+          assert control? "\x1E"
+          refute control? "\x67"
+        end
+
+        it "is blank or not" do
+          assert blank? "\x20"
+          refute blank? "\x21"
+        end
+
+        it "is never format" do
+          refute format? "\x21"
+        end
+      end
+
+      # describe "macCentEuro" do
+      # describe "macCroatian" do
+      # describe "macCyrillic" do
+      # describe "macGreek" do
+      # describe "macIceland" do
+      # describe "macRomania" do
+      # describe "macThai" do
+      # describe "macTurkish" do
+      # describe "macUkraine" do
+    end
+
+    describe "TIS-620/Windows-874" do
+      describe "TIS-620" do
+        let(:encoding) { "TIS-620" }
+
+        it "is always valid" do
+          assert valid? "\x80"
+        end
+
+        it "is assigned or not" do
+          assert assigned? "\x21"
+          refute assigned? "\xA0"
+        end
+
+        it "is control or not" do
+          assert control? "\x1E"
+          refute control? "\x67"
+        end
+
+        it "is blank or not" do
+          assert blank? "\x20"
+          refute blank? "\x21"
+        end
+
+        it "is never format" do
+          refute format? "\x21"
+        end
+      end
+
+      describe "Windows-874" do
+        let(:encoding) { "Windows-874" }
+
+        it "is always valid" do
+          assert valid? "\x80"
+        end
+
+        it "is assigned or not" do
+          assert assigned? "\xA0"
+          refute assigned? "\x99"
+        end
+
+        it "is control or not" do
+          assert control? "\x1E"
+          refute control? "\x67"
+        end
+
+        it "is blank or not" do
+          assert blank? "\x20"
+          refute blank? "\x21"
+        end
+
+        it "is never format" do
+          refute format? "\x21"
+        end
+      end
+    end
+
+    describe "KOI8-*" do
+      describe "KOI8-R" do
+        let(:encoding) { "KOI8-R" }
+
+        it "is always valid" do
+          assert valid? "\x80"
+        end
+
+        it "is always assigned" do
+          assert assigned? "\x21"
+        end
+
+        it "is control or not" do
+          assert control? "\x1E"
+          refute control? "\x67"
+        end
+
+        it "is blank or not" do
+          assert blank? "\x20"
+          refute blank? "\x21"
+        end
+
+        it "is never format" do
+          refute format? "\x21"
+        end
+      end
+
+      describe "KOI8-U" do
+        let(:encoding) { "KOI8-U" }
+
+        it "is always valid" do
+          assert valid? "\x80"
+        end
+
+        it "is always assigned" do
+          assert assigned? "\x21"
+        end
+
+        it "is control or not" do
+          assert control? "\x1E"
+          refute control? "\x67"
+        end
+
+        it "is blank or not" do
+          assert blank? "\x20"
+          refute blank? "\x21"
+        end
+
+        it "is never format" do
+          refute format? "\x21"
+        end
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: characteristics
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Jan Lelis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-13 00:00:00.000000000 Z
+date: 2017-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode-categories