unibits 1.3.0 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (9) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/README.md +14 -3
  4. data/lib/unibits.rb +42 -33
  5. data/lib/unibits/symbolify.rb +419 -328
  6. data/lib/unibits/version.rb +1 -1
  7. data/spec/unibits_spec.rb +18 -0
  8. data/unibits.gemspec +1 -1
  9. metadata +5 -11
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b811d8983c85bc87c33a59b4cb7cd5144104a994
4
- data.tar.gz: 63cf6f3b91f26b8ac8b0fca9d90a595662a9f0f4
3
+ metadata.gz: 0d5f49745b4dd8f80a9b4c87c3810b4ddc5dbe66
4
+ data.tar.gz: a00bc08a018d1c1e40d9560f48cf64e53aa4e20b
5
5
  SHA512:
6
- metadata.gz: 65979ed1477c6f5bcbed7b6f74474c1958e582614d91ae0a2ebf5cdf605b31f4263b1cc68e1ff113b21ce572c7025bfacf85db5353b9108e07cf352a3c60e052
7
- data.tar.gz: b4cfbad99ad6257616b96e95f53f21696e545b4b29fafd6c3e212e03bc51f52eddb66c06ddb0b1fe2842a42cb5553f5d1a5ca32b7320aa463cd7bbf9de4f0ad3
6
+ metadata.gz: 865101498c37b0d480846c6eb9a613942becef9f0f0fa02d01ed78ca1dc8bd4d7d3ae10ce3856ab76ded888278e27da6962657de65ea96c464920c5e3c91866d
7
+ data.tar.gz: 39362d02cc5f8b8f0c6df3cc3b7f1bb597eef0eb8e2fb828b16f95de8e6a8e8b41faa13d03a33fcb1d2890b7625a855a75924351dca76832994cf50d1af95e9f
data/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  ## CHANGELOG
2
2
 
3
+ ### 2.0.0
4
+
5
+ * Support more encodings: ISO-8859-X and Windows-125X
6
+ * Add three HANGUL characters (U+115F, U+1160, U+3164) to list of possible white spaces
7
+ * Move character handling to separate gem. It is called "characteristics".
8
+ * Highlight control chars in blue and blanks in light blue
9
+ * Handle encodings that are not convertible to UTF-8
10
+
3
11
  ### 1.3.0
4
12
 
5
13
  * Add variation selectors 17-256 (U+E0100 - U+E01EF)
data/README.md CHANGED
@@ -4,9 +4,19 @@ Ruby library and CLI command that visualizes various Unicode and ASCII encodings
4
4
 
5
5
  - Makes analyzing encodings easier
6
6
  - Helps you with debugging strings
7
- - Supports **UTF-8**, **UTF-16LE**/**UTF-16BE**, **UTF-32LE**/**UTF-32BE**, arbitrary **BINARY** data, and **ASCII**
7
+ - Supports **UTF-8**, **UTF-16LE**/**UTF-16BE**, **UTF-32LE**/**UTF-32BE**, **ISO-8859-X**, **Windows-125X** arbitrary **BINARY** data, and **ASCII**
8
8
  - Highlights invalid encodings
9
9
 
10
+ ## Color Coding
11
+
12
+ Each byte of the given string is highlighted using the following mechanism:
13
+
14
+ - Red for invalid bytes
15
+ - Orange for unassigned bytes/characters
16
+ - Blue for control characters
17
+ - Light blue for blanks
18
+ - Random color for all other characters
19
+
10
20
  ## Setup
11
21
 
12
22
  Make sure you have Ruby installed and installing gems works properly. Then do:
@@ -114,16 +124,17 @@ Example in Ruby: `unibits "🌫 Idiosyncrätic ℜսᖯʏ", encoding: 'ascii'
114
124
 
115
125
  ### BINARY
116
126
 
117
- (not possible to produce invalid binary strings)
127
+ Not possible to produce invalid binary strings
118
128
 
119
129
  ## Notes
120
130
 
121
131
  Also see
122
132
 
133
+ - [Ruby's Encoding class](https://ruby-doc.org/core/Encoding.html)
134
+ - [Characteristics gem](https://github.com/janlelis/characteristics)
123
135
  - [UTF-8 (Wikipedia)](https://en.wikipedia.org/wiki/UTF-8#Description)
124
136
  - [UTF-16 (Wikipedia)](https://en.wikipedia.org/wiki/UTF-16#Description)
125
137
  - [UTF-32 (Wikipedia)](https://en.wikipedia.org/wiki/UTF-32)
126
- - [Ruby's Encoding class](https://ruby-doc.org/core/Encoding.html)
127
138
  - [Difference between BINARY and ASCII](http://idiosyncratic-ruby.com/56-us-ascii-8bit.html)
128
139
  - [Unicode Micro Libraries for Ruby](https://github.com/janlelis/unicode-x)
129
140
 
data/lib/unibits.rb CHANGED
@@ -4,7 +4,7 @@ require_relative "unibits/symbolify"
4
4
  require "io/console"
5
5
  require "paint"
6
6
  require "unicode/display_width"
7
- require "unicode/categories"
7
+ require "characteristics"
8
8
 
9
9
  module Unibits
10
10
  SUPPORTED_ENCODINGS = [
@@ -15,6 +15,8 @@ module Unibits
15
15
  'UTF-32BE',
16
16
  'ASCII-8BIT',
17
17
  'US-ASCII',
18
+ /^ISO-8859-/,
19
+ /^Windows-125/,
18
20
  ].freeze
19
21
  DEFAULT_TERMINAL_WIDTH = 80
20
22
 
@@ -49,6 +51,9 @@ module Unibits
49
51
 
50
52
  def self.visualize(string, wide_ambiguous: false, width: nil)
51
53
  cols = width || determine_terminal_cols
54
+ encoding_name = string.encoding.name
55
+
56
+ type = Characteristics.type_from_encoding_name(encoding_name)
52
57
 
53
58
  cp_buffer = [" "]
54
59
  enc_buffer = [" "]
@@ -59,18 +64,10 @@ module Unibits
59
64
 
60
65
  puts
61
66
  string.each_char{ |char|
62
- if char.valid_encoding?
63
- char_valid = true
64
- current_encoding_error = nil
65
- if Unicode::Categories.category(char) == "Cn"
66
- current_color = "#FF5500"
67
- else
68
- current_color = random_color
69
- end
70
- else
71
- char_valid = false
72
- current_color = :red
73
- end
67
+ char_info = Characteristics.create_for_type(char, type)
68
+ current_color = determine_char_color(char_info)
69
+
70
+ current_encoding_error = nil if char_info.valid?
74
71
 
75
72
  char.each_byte.with_index{ |byte, index|
76
73
  if Paint.unpaint(hex_buffer[-1]).bytesize > cols - 12
@@ -82,10 +79,10 @@ module Unibits
82
79
  end
83
80
 
84
81
  if index == 0
85
- if char_valid
82
+ if char_info.valid?
86
83
  codepoint = "U+%04X" % char.ord
87
84
  else
88
- case string.encoding.name
85
+ case encoding_name
89
86
  when "US-ASCII"
90
87
  codepoint = "invalid"
91
88
  when "UTF-8"
@@ -167,7 +164,7 @@ module Unibits
167
164
  when 'UTF-16LE', 'UTF-16BE'
168
165
  if char.bytesize.odd?
169
166
  codepoint = "incompl."
170
- elsif char.b[string.encoding.name == 'UTF-16LE' ? 1 : 0].unpack("B*")[0][0, 5] == "11011"
167
+ elsif char.b[encoding_name == 'UTF-16LE' ? 1 : 0].unpack("B*")[0][0, 5] == "11011"
171
168
  codepoint = "hlf.srg."
172
169
  else
173
170
  codepoint = "invalid"
@@ -185,14 +182,14 @@ module Unibits
185
182
  codepoint.ljust(10), current_color, :bold
186
183
  ]
187
184
 
188
- if char_valid
189
- symbolified_char = symbolify(char)
185
+ symbolified_char = Symbolify.symbolify(char, char_info)
186
+
187
+ if char_info.unicode?
188
+ padding = 10 - Unicode::DisplayWidth.of(symbolified_char, wide_ambiguous ? 2 : 1)
190
189
  else
191
- symbolified_char = "�"
190
+ padding = 10 - symbolified_char.size
192
191
  end
193
192
 
194
- padding = 10 - Unicode::DisplayWidth.of(symbolified_char, wide_ambiguous ? 2 : 1)
195
-
196
193
  enc_buffer[-1] << Paint[
197
194
  symbolified_char, current_color
198
195
  ]
@@ -208,11 +205,11 @@ module Unibits
208
205
 
209
206
  bin_byte_complete = byte.to_s(2).rjust(8, "0")
210
207
 
211
- if !char_valid
208
+ if !char_info.valid?
212
209
  bin_byte_1 = bin_byte_complete
213
210
  bin_byte_2 = ""
214
211
  else
215
- case string.encoding.name
212
+ case encoding_name
216
213
  when 'US-ASCII'
217
214
  bin_byte_1 = bin_byte_complete[0...1]
218
215
  bin_byte_2 = bin_byte_complete[1...8]
@@ -253,6 +250,9 @@ module Unibits
253
250
  when 'UTF-32LE', 'UTF-32BE'
254
251
  bin_byte_1 = ""
255
252
  bin_byte_2 = bin_byte_complete
253
+ when /^(ISO-8859-|Windows-125)/
254
+ bin_byte_1 = ""
255
+ bin_byte_2 = bin_byte_complete
256
256
  end
257
257
  end
258
258
 
@@ -268,25 +268,34 @@ module Unibits
268
268
  }
269
269
  }
270
270
 
271
- if string.encoding.name[0, 3] == "UTF"
271
+ if type == :unicode
272
272
  enc_buffer.zip(cp_buffer, hex_buffer, bin_buffer, separator).flatten.join("\n")
273
273
  else
274
274
  enc_buffer.zip(hex_buffer, bin_buffer, separator).flatten.join("\n")
275
275
  end
276
276
  end
277
277
 
278
- def self.random_color
279
- "%.2x%.2x%.2x" %[rand(90) + 60, rand(90) + 60, rand(90) + 60]
280
- end
281
-
282
- def self.symbolify(char)
283
- return char.inspect unless char.encoding.name[0, 3] == "UTF"
284
- Symbolify.symbolify(char).encode('UTF-8')
285
- end
286
-
287
278
  def self.determine_terminal_cols
288
279
  STDIN.winsize[1] || DEFAULT_TERMINAL_WIDTH
289
280
  rescue Errno::ENOTTY
290
281
  return DEFAULT_TERMINAL_WIDTH
291
282
  end
283
+
284
+ def self.determine_char_color(char_info)
285
+ if !char_info.valid?
286
+ "#FF0000"
287
+ elsif !char_info.assigned?
288
+ "#FF5500"
289
+ elsif char_info.control?
290
+ "#0000FF"
291
+ elsif char_info.blank?
292
+ "#33AADD"
293
+ else
294
+ random_color
295
+ end
296
+ end
297
+
298
+ def self.random_color
299
+ "%.2x%.2x%.2x" % [rand(90) + 60, rand(90) + 60, rand(90) + 60]
300
+ end
292
301
  end
data/lib/unibits/symbolify.rb CHANGED
@@ -1,351 +1,442 @@
1
- require "unicode/categories"
2
-
3
1
  module Unibits
4
2
  module Symbolify
3
+ NO_UTF8_CONVERTER = /^Windows-1258/
4
+ ASCII_CHARS = "\x20-\x7E".freeze
5
5
  ASCII_CONTROL_CODEPOINTS = "\x00-\x1F\x7F".freeze
6
6
  ASCII_CONTROL_SYMBOLS = "\u{2400}-\u{241F}\u{2421}".freeze
7
- ASCII_CHARS = "\x20-\x7E".freeze
8
- TAG_START = "\u{E0001}".freeze
9
- TAG_START_SYMBOL = "LANG TAG".freeze
10
- TAG_SPACE = "\u{E0020}".freeze
11
- TAG_SPACE_SYMBOL = "TAG ␠".freeze
12
7
  TAGS = "\u{E0021}-\u{E007E}".freeze
13
- TAG_DELETE = "\u{E007F}".freeze
14
- TAG_DELETE_SYMBOL = "TAG ␡".freeze
8
+
9
+ CONTROL_C0_SYMBOLS = [
10
+ "␀",
11
+ "␁",
12
+ "␂",
13
+ "␃",
14
+ "␄",
15
+ "␅",
16
+ "␆",
17
+ "␇",
18
+ "␈",
19
+ "␉",
20
+ "␊",
21
+ "␋",
22
+ "␌",
23
+ "␍",
24
+ "␎",
25
+ "␏",
26
+ "␐",
27
+ "␑",
28
+ "␒",
29
+ "␓",
30
+ "␔",
31
+ "␕",
32
+ "␖",
33
+ "␗",
34
+ "␘",
35
+ "␙",
36
+ "␚",
37
+ "␛",
38
+ "␜",
39
+ "␝",
40
+ "␞",
41
+ "␟",
42
+ ]
43
+
44
+ CONTROL_DELETE_SYMBOL = "␡"
45
+
46
+ CONTROL_C1_NAMES = {
47
+ 0x80 => "PAD",
48
+ 0x81 => "HOP",
49
+ 0x82 => "BPH",
50
+ 0x83 => "NBH",
51
+ 0x84 => "IND",
52
+ 0x85 => "NEL",
53
+ 0x86 => "SSA",
54
+ 0x87 => "ESA",
55
+ 0x88 => "HTS",
56
+ 0x89 => "HTJ",
57
+ 0x8A => "VTS",
58
+ 0x8B => "PLD",
59
+ 0x8C => "PLU",
60
+ 0x8D => "RI",
61
+ 0x8E => "SS2",
62
+ 0x8F => "SS3",
63
+ 0x90 => "DCS",
64
+ 0x91 => "PU1",
65
+ 0x92 => "PU2",
66
+ 0x93 => "STS",
67
+ 0x94 => "CCH",
68
+ 0x95 => "MW",
69
+ 0x96 => "SPA",
70
+ 0x97 => "EPA",
71
+ 0x98 => "SOS",
72
+ 0x99 => "SGC",
73
+ 0x9A => "SCI",
74
+ 0x9B => "CSI",
75
+ 0x9C => "ST",
76
+ 0x9D => "OSC",
77
+ 0x9E => "PM",
78
+ 0x9F => "APC",
79
+ }
80
+
15
81
  INTERESTING_CODEPOINTS = {
16
- "\u{0080}" => "PAD",
17
- "\u{0081}" => "HOP",
18
- "\u{0082}" => "BPH",
19
- "\u{0083}" => "NBH",
20
- "\u{0084}" => "IND",
21
- "\u{0085}" => "NEL",
22
- "\u{0086}" => "SSA",
23
- "\u{0087}" => "ESA",
24
- "\u{0088}" => "HTS",
25
- "\u{0089}" => "HTJ",
26
- "\u{008A}" => "VTS",
27
- "\u{008B}" => "PLD",
28
- "\u{008C}" => "PLU",
29
- "\u{008D}" => "RI",
30
- "\u{008E}" => "SS2",
31
- "\u{008F}" => "SS3",
32
- "\u{0090}" => "DCS",
33
- "\u{0091}" => "PU1",
34
- "\u{0092}" => "PU2",
35
- "\u{0093}" => "STS",
36
- "\u{0094}" => "CCH",
37
- "\u{0095}" => "MW",
38
- "\u{0096}" => "SPA",
39
- "\u{0097}" => "EPA",
40
- "\u{0098}" => "SOS",
41
- "\u{0099}" => "SGC",
42
- "\u{009A}" => "SCI",
43
- "\u{009B}" => "CSI",
44
- "\u{009C}" => "ST",
45
- "\u{009D}" => "OSC",
46
- "\u{009E}" => "PM",
47
- "\u{009F}" => "APC",
82
+ 0x200E => "LRM",
83
+ 0x200F => "RLM",
84
+ 0x202A => "LRE",
85
+ 0x202B => "RLE",
86
+ 0x202C => "PDF",
87
+ 0x202D => "LRO",
88
+ 0x202E => "RLO",
89
+ 0x2066 => "LRI",
90
+ 0x2067 => "RLI",
91
+ 0x2068 => "FSI",
92
+ 0x2069 => "PDI",
48
93
 
49
- "\u{200E}" => "LRM",
50
- "\u{200F}" => "RLM",
51
- "\u{202A}" => "LRE",
52
- "\u{202B}" => "RLE",
53
- "\u{202C}" => "PDF",
54
- "\u{202D}" => "LRO",
55
- "\u{202E}" => "RLO",
56
- "\u{2066}" => "LRI",
57
- "\u{2067}" => "RLI",
58
- "\u{2068}" => "FSI",
59
- "\u{2069}" => "PDI",
94
+ 0xFE00 => "VS1",
95
+ 0xFE01 => "VS2",
96
+ 0xFE02 => "VS3",
97
+ 0xFE03 => "VS4",
98
+ 0xFE04 => "VS5",
99
+ 0xFE05 => "VS6",
100
+ 0xFE06 => "VS7",
101
+ 0xFE07 => "VS8",
102
+ 0xFE08 => "VS9",
103
+ 0xFE09 => "VS10",
104
+ 0xFE0A => "VS11",
105
+ 0xFE0B => "VS12",
106
+ 0xFE0C => "VS13",
107
+ 0xFE0D => "VS14",
108
+ 0xFE0E => "VS15",
109
+ 0xFE0F => "VS16",
60
110
 
61
- "\u{FE00}" => "VS1",
62
- "\u{FE01}" => "VS2",
63
- "\u{FE02}" => "VS3",
64
- "\u{FE03}" => "VS4",
65
- "\u{FE04}" => "VS5",
66
- "\u{FE05}" => "VS6",
67
- "\u{FE06}" => "VS7",
68
- "\u{FE07}" => "VS8",
69
- "\u{FE08}" => "VS9",
70
- "\u{FE09}" => "VS10",
71
- "\u{FE0A}" => "VS11",
72
- "\u{FE0B}" => "VS12",
73
- "\u{FE0C}" => "VS13",
74
- "\u{FE0D}" => "VS14",
75
- "\u{FE0E}" => "VS15",
76
- "\u{FE0F}" => "VS16",
111
+ 0xE0001 => "LANG TAG",
112
+ 0xE0020 => "TAG ␠",
113
+ 0xE007F => "TAG ␡",
77
114
 
78
- "\u{E0100}" => "VS17",
79
- "\u{E0101}" => "VS18",
80
- "\u{E0102}" => "VS19",
81
- "\u{E0103}" => "VS20",
82
- "\u{E0104}" => "VS21",
83
- "\u{E0105}" => "VS22",
84
- "\u{E0106}" => "VS23",
85
- "\u{E0107}" => "VS24",
86
- "\u{E0108}" => "VS25",
87
- "\u{E0109}" => "VS26",
88
- "\u{E010A}" => "VS27",
89
- "\u{E010B}" => "VS28",
90
- "\u{E010C}" => "VS29",
91
- "\u{E010D}" => "VS30",
92
- "\u{E010E}" => "VS31",
93
- "\u{E010F}" => "VS32",
94
- "\u{E0110}" => "VS33",
95
- "\u{E0111}" => "VS34",
96
- "\u{E0112}" => "VS35",
97
- "\u{E0113}" => "VS36",
98
- "\u{E0114}" => "VS37",
99
- "\u{E0115}" => "VS38",
100
- "\u{E0116}" => "VS39",
101
- "\u{E0117}" => "VS40",
102
- "\u{E0118}" => "VS41",
103
- "\u{E0119}" => "VS42",
104
- "\u{E011A}" => "VS43",
105
- "\u{E011B}" => "VS44",
106
- "\u{E011C}" => "VS45",
107
- "\u{E011D}" => "VS46",
108
- "\u{E011E}" => "VS47",
109
- "\u{E011F}" => "VS48",
110
- "\u{E0120}" => "VS49",
111
- "\u{E0121}" => "VS50",
112
- "\u{E0122}" => "VS51",
113
- "\u{E0123}" => "VS52",
114
- "\u{E0124}" => "VS53",
115
- "\u{E0125}" => "VS54",
116
- "\u{E0126}" => "VS55",
117
- "\u{E0127}" => "VS56",
118
- "\u{E0128}" => "VS57",
119
- "\u{E0129}" => "VS58",
120
- "\u{E012A}" => "VS59",
121
- "\u{E012B}" => "VS60",
122
- "\u{E012C}" => "VS61",
123
- "\u{E012D}" => "VS62",
124
- "\u{E012E}" => "VS63",
125
- "\u{E012F}" => "VS64",
126
- "\u{E0130}" => "VS65",
127
- "\u{E0131}" => "VS66",
128
- "\u{E0132}" => "VS67",
129
- "\u{E0133}" => "VS68",
130
- "\u{E0134}" => "VS69",
131
- "\u{E0135}" => "VS70",
132
- "\u{E0136}" => "VS71",
133
- "\u{E0137}" => "VS72",
134
- "\u{E0138}" => "VS73",
135
- "\u{E0139}" => "VS74",
136
- "\u{E013A}" => "VS75",
137
- "\u{E013B}" => "VS76",
138
- "\u{E013C}" => "VS77",
139
- "\u{E013D}" => "VS78",
140
- "\u{E013E}" => "VS79",
141
- "\u{E013F}" => "VS80",
142
- "\u{E0140}" => "VS81",
143
- "\u{E0141}" => "VS82",
144
- "\u{E0142}" => "VS83",
145
- "\u{E0143}" => "VS84",
146
- "\u{E0144}" => "VS85",
147
- "\u{E0145}" => "VS86",
148
- "\u{E0146}" => "VS87",
149
- "\u{E0147}" => "VS88",
150
- "\u{E0148}" => "VS89",
151
- "\u{E0149}" => "VS90",
152
- "\u{E014A}" => "VS91",
153
- "\u{E014B}" => "VS92",
154
- "\u{E014C}" => "VS93",
155
- "\u{E014D}" => "VS94",
156
- "\u{E014E}" => "VS95",
157
- "\u{E014F}" => "VS96",
158
- "\u{E0150}" => "VS97",
159
- "\u{E0151}" => "VS98",
160
- "\u{E0152}" => "VS99",
161
- "\u{E0153}" => "VS100",
162
- "\u{E0154}" => "VS101",
163
- "\u{E0155}" => "VS102",
164
- "\u{E0156}" => "VS103",
165
- "\u{E0157}" => "VS104",
166
- "\u{E0158}" => "VS105",
167
- "\u{E0159}" => "VS106",
168
- "\u{E015A}" => "VS107",
169
- "\u{E015B}" => "VS108",
170
- "\u{E015C}" => "VS109",
171
- "\u{E015D}" => "VS110",
172
- "\u{E015E}" => "VS111",
173
- "\u{E015F}" => "VS112",
174
- "\u{E0160}" => "VS113",
175
- "\u{E0161}" => "VS114",
176
- "\u{E0162}" => "VS115",
177
- "\u{E0163}" => "VS116",
178
- "\u{E0164}" => "VS117",
179
- "\u{E0165}" => "VS118",
180
- "\u{E0166}" => "VS119",
181
- "\u{E0167}" => "VS120",
182
- "\u{E0168}" => "VS121",
183
- "\u{E0169}" => "VS122",
184
- "\u{E016A}" => "VS123",
185
- "\u{E016B}" => "VS124",
186
- "\u{E016C}" => "VS125",
187
- "\u{E016D}" => "VS126",
188
- "\u{E016E}" => "VS127",
189
- "\u{E016F}" => "VS128",
190
- "\u{E0170}" => "VS129",
191
- "\u{E0171}" => "VS130",
192
- "\u{E0172}" => "VS131",
193
- "\u{E0173}" => "VS132",
194
- "\u{E0174}" => "VS133",
195
- "\u{E0175}" => "VS134",
196
- "\u{E0176}" => "VS135",
197
- "\u{E0177}" => "VS136",
198
- "\u{E0178}" => "VS137",
199
- "\u{E0179}" => "VS138",
200
- "\u{E017A}" => "VS139",
201
- "\u{E017B}" => "VS140",
202
- "\u{E017C}" => "VS141",
203
- "\u{E017D}" => "VS142",
204
- "\u{E017E}" => "VS143",
205
- "\u{E017F}" => "VS144",
206
- "\u{E0180}" => "VS145",
207
- "\u{E0181}" => "VS146",
208
- "\u{E0182}" => "VS147",
209
- "\u{E0183}" => "VS148",
210
- "\u{E0184}" => "VS149",
211
- "\u{E0185}" => "VS150",
212
- "\u{E0186}" => "VS151",
213
- "\u{E0187}" => "VS152",
214
- "\u{E0188}" => "VS153",
215
- "\u{E0189}" => "VS154",
216
- "\u{E018A}" => "VS155",
217
- "\u{E018B}" => "VS156",
218
- "\u{E018C}" => "VS157",
219
- "\u{E018D}" => "VS158",
220
- "\u{E018E}" => "VS159",
221
- "\u{E018F}" => "VS160",
222
- "\u{E0190}" => "VS161",
223
- "\u{E0191}" => "VS162",
224
- "\u{E0192}" => "VS163",
225
- "\u{E0193}" => "VS164",
226
- "\u{E0194}" => "VS165",
227
- "\u{E0195}" => "VS166",
228
- "\u{E0196}" => "VS167",
229
- "\u{E0197}" => "VS168",
230
- "\u{E0198}" => "VS169",
231
- "\u{E0199}" => "VS170",
232
- "\u{E019A}" => "VS171",
233
- "\u{E019B}" => "VS172",
234
- "\u{E019C}" => "VS173",
235
- "\u{E019D}" => "VS174",
236
- "\u{E019E}" => "VS175",
237
- "\u{E019F}" => "VS176",
238
- "\u{E01A0}" => "VS177",
239
- "\u{E01A1}" => "VS178",
240
- "\u{E01A2}" => "VS179",
241
- "\u{E01A3}" => "VS180",
242
- "\u{E01A4}" => "VS181",
243
- "\u{E01A5}" => "VS182",
244
- "\u{E01A6}" => "VS183",
245
- "\u{E01A7}" => "VS184",
246
- "\u{E01A8}" => "VS185",
247
- "\u{E01A9}" => "VS186",
248
- "\u{E01AA}" => "VS187",
249
- "\u{E01AB}" => "VS188",
250
- "\u{E01AC}" => "VS189",
251
- "\u{E01AD}" => "VS190",
252
- "\u{E01AE}" => "VS191",
253
- "\u{E01AF}" => "VS192",
254
- "\u{E01B0}" => "VS193",
255
- "\u{E01B1}" => "VS194",
256
- "\u{E01B2}" => "VS195",
257
- "\u{E01B3}" => "VS196",
258
- "\u{E01B4}" => "VS197",
259
- "\u{E01B5}" => "VS198",
260
- "\u{E01B6}" => "VS199",
261
- "\u{E01B7}" => "VS200",
262
- "\u{E01B8}" => "VS201",
263
- "\u{E01B9}" => "VS202",
264
- "\u{E01BA}" => "VS203",
265
- "\u{E01BB}" => "VS204",
266
- "\u{E01BC}" => "VS205",
267
- "\u{E01BD}" => "VS206",
268
- "\u{E01BE}" => "VS207",
269
- "\u{E01BF}" => "VS208",
270
- "\u{E01C0}" => "VS209",
271
- "\u{E01C1}" => "VS210",
272
- "\u{E01C2}" => "VS211",
273
- "\u{E01C3}" => "VS212",
274
- "\u{E01C4}" => "VS213",
275
- "\u{E01C5}" => "VS214",
276
- "\u{E01C6}" => "VS215",
277
- "\u{E01C7}" => "VS216",
278
- "\u{E01C8}" => "VS217",
279
- "\u{E01C9}" => "VS218",
280
- "\u{E01CA}" => "VS219",
281
- "\u{E01CB}" => "VS220",
282
- "\u{E01CC}" => "VS221",
283
- "\u{E01CD}" => "VS222",
284
- "\u{E01CE}" => "VS223",
285
- "\u{E01CF}" => "VS224",
286
- "\u{E01D0}" => "VS225",
287
- "\u{E01D1}" => "VS226",
288
- "\u{E01D2}" => "VS227",
289
- "\u{E01D3}" => "VS228",
290
- "\u{E01D4}" => "VS229",
291
- "\u{E01D5}" => "VS230",
292
- "\u{E01D6}" => "VS231",
293
- "\u{E01D7}" => "VS232",
294
- "\u{E01D8}" => "VS233",
295
- "\u{E01D9}" => "VS234",
296
- "\u{E01DA}" => "VS235",
297
- "\u{E01DB}" => "VS236",
298
- "\u{E01DC}" => "VS237",
299
- "\u{E01DD}" => "VS238",
300
- "\u{E01DE}" => "VS239",
301
- "\u{E01DF}" => "VS240",
302
- "\u{E01E0}" => "VS241",
303
- "\u{E01E1}" => "VS242",
304
- "\u{E01E2}" => "VS243",
305
- "\u{E01E3}" => "VS244",
306
- "\u{E01E4}" => "VS245",
307
- "\u{E01E5}" => "VS246",
308
- "\u{E01E6}" => "VS247",
309
- "\u{E01E7}" => "VS248",
310
- "\u{E01E8}" => "VS249",
311
- "\u{E01E9}" => "VS250",
312
- "\u{E01EA}" => "VS251",
313
- "\u{E01EB}" => "VS252",
314
- "\u{E01EC}" => "VS253",
315
- "\u{E01ED}" => "VS254",
316
- "\u{E01EE}" => "VS255",
317
- "\u{E01EF}" => "VS256",
115
+ 0xE0100 => "VS17",
116
+ 0xE0101 => "VS18",
117
+ 0xE0102 => "VS19",
118
+ 0xE0103 => "VS20",
119
+ 0xE0104 => "VS21",
120
+ 0xE0105 => "VS22",
121
+ 0xE0106 => "VS23",
122
+ 0xE0107 => "VS24",
123
+ 0xE0108 => "VS25",
124
+ 0xE0109 => "VS26",
125
+ 0xE010A => "VS27",
126
+ 0xE010B => "VS28",
127
+ 0xE010C => "VS29",
128
+ 0xE010D => "VS30",
129
+ 0xE010E => "VS31",
130
+ 0xE010F => "VS32",
131
+ 0xE0110 => "VS33",
132
+ 0xE0111 => "VS34",
133
+ 0xE0112 => "VS35",
134
+ 0xE0113 => "VS36",
135
+ 0xE0114 => "VS37",
136
+ 0xE0115 => "VS38",
137
+ 0xE0116 => "VS39",
138
+ 0xE0117 => "VS40",
139
+ 0xE0118 => "VS41",
140
+ 0xE0119 => "VS42",
141
+ 0xE011A => "VS43",
142
+ 0xE011B => "VS44",
143
+ 0xE011C => "VS45",
144
+ 0xE011D => "VS46",
145
+ 0xE011E => "VS47",
146
+ 0xE011F => "VS48",
147
+ 0xE0120 => "VS49",
148
+ 0xE0121 => "VS50",
149
+ 0xE0122 => "VS51",
150
+ 0xE0123 => "VS52",
151
+ 0xE0124 => "VS53",
152
+ 0xE0125 => "VS54",
153
+ 0xE0126 => "VS55",
154
+ 0xE0127 => "VS56",
155
+ 0xE0128 => "VS57",
156
+ 0xE0129 => "VS58",
157
+ 0xE012A => "VS59",
158
+ 0xE012B => "VS60",
159
+ 0xE012C => "VS61",
160
+ 0xE012D => "VS62",
161
+ 0xE012E => "VS63",
162
+ 0xE012F => "VS64",
163
+ 0xE0130 => "VS65",
164
+ 0xE0131 => "VS66",
165
+ 0xE0132 => "VS67",
166
+ 0xE0133 => "VS68",
167
+ 0xE0134 => "VS69",
168
+ 0xE0135 => "VS70",
169
+ 0xE0136 => "VS71",
170
+ 0xE0137 => "VS72",
171
+ 0xE0138 => "VS73",
172
+ 0xE0139 => "VS74",
173
+ 0xE013A => "VS75",
174
+ 0xE013B => "VS76",
175
+ 0xE013C => "VS77",
176
+ 0xE013D => "VS78",
177
+ 0xE013E => "VS79",
178
+ 0xE013F => "VS80",
179
+ 0xE0140 => "VS81",
180
+ 0xE0141 => "VS82",
181
+ 0xE0142 => "VS83",
182
+ 0xE0143 => "VS84",
183
+ 0xE0144 => "VS85",
184
+ 0xE0145 => "VS86",
185
+ 0xE0146 => "VS87",
186
+ 0xE0147 => "VS88",
187
+ 0xE0148 => "VS89",
188
+ 0xE0149 => "VS90",
189
+ 0xE014A => "VS91",
190
+ 0xE014B => "VS92",
191
+ 0xE014C => "VS93",
192
+ 0xE014D => "VS94",
193
+ 0xE014E => "VS95",
194
+ 0xE014F => "VS96",
195
+ 0xE0150 => "VS97",
196
+ 0xE0151 => "VS98",
197
+ 0xE0152 => "VS99",
198
+ 0xE0153 => "VS100",
199
+ 0xE0154 => "VS101",
200
+ 0xE0155 => "VS102",
201
+ 0xE0156 => "VS103",
202
+ 0xE0157 => "VS104",
203
+ 0xE0158 => "VS105",
204
+ 0xE0159 => "VS106",
205
+ 0xE015A => "VS107",
206
+ 0xE015B => "VS108",
207
+ 0xE015C => "VS109",
208
+ 0xE015D => "VS110",
209
+ 0xE015E => "VS111",
210
+ 0xE015F => "VS112",
211
+ 0xE0160 => "VS113",
212
+ 0xE0161 => "VS114",
213
+ 0xE0162 => "VS115",
214
+ 0xE0163 => "VS116",
215
+ 0xE0164 => "VS117",
216
+ 0xE0165 => "VS118",
217
+ 0xE0166 => "VS119",
218
+ 0xE0167 => "VS120",
219
+ 0xE0168 => "VS121",
220
+ 0xE0169 => "VS122",
221
+ 0xE016A => "VS123",
222
+ 0xE016B => "VS124",
223
+ 0xE016C => "VS125",
224
+ 0xE016D => "VS126",
225
+ 0xE016E => "VS127",
226
+ 0xE016F => "VS128",
227
+ 0xE0170 => "VS129",
228
+ 0xE0171 => "VS130",
229
+ 0xE0172 => "VS131",
230
+ 0xE0173 => "VS132",
231
+ 0xE0174 => "VS133",
232
+ 0xE0175 => "VS134",
233
+ 0xE0176 => "VS135",
234
+ 0xE0177 => "VS136",
235
+ 0xE0178 => "VS137",
236
+ 0xE0179 => "VS138",
237
+ 0xE017A => "VS139",
238
+ 0xE017B => "VS140",
239
+ 0xE017C => "VS141",
240
+ 0xE017D => "VS142",
241
+ 0xE017E => "VS143",
242
+ 0xE017F => "VS144",
243
+ 0xE0180 => "VS145",
244
+ 0xE0181 => "VS146",
245
+ 0xE0182 => "VS147",
246
+ 0xE0183 => "VS148",
247
+ 0xE0184 => "VS149",
248
+ 0xE0185 => "VS150",
249
+ 0xE0186 => "VS151",
250
+ 0xE0187 => "VS152",
251
+ 0xE0188 => "VS153",
252
+ 0xE0189 => "VS154",
253
+ 0xE018A => "VS155",
254
+ 0xE018B => "VS156",
255
+ 0xE018C => "VS157",
256
+ 0xE018D => "VS158",
257
+ 0xE018E => "VS159",
258
+ 0xE018F => "VS160",
259
+ 0xE0190 => "VS161",
260
+ 0xE0191 => "VS162",
261
+ 0xE0192 => "VS163",
262
+ 0xE0193 => "VS164",
263
+ 0xE0194 => "VS165",
264
+ 0xE0195 => "VS166",
265
+ 0xE0196 => "VS167",
266
+ 0xE0197 => "VS168",
267
+ 0xE0198 => "VS169",
268
+ 0xE0199 => "VS170",
269
+ 0xE019A => "VS171",
270
+ 0xE019B => "VS172",
271
+ 0xE019C => "VS173",
272
+ 0xE019D => "VS174",
273
+ 0xE019E => "VS175",
274
+ 0xE019F => "VS176",
275
+ 0xE01A0 => "VS177",
276
+ 0xE01A1 => "VS178",
277
+ 0xE01A2 => "VS179",
278
+ 0xE01A3 => "VS180",
279
+ 0xE01A4 => "VS181",
280
+ 0xE01A5 => "VS182",
281
+ 0xE01A6 => "VS183",
282
+ 0xE01A7 => "VS184",
283
+ 0xE01A8 => "VS185",
284
+ 0xE01A9 => "VS186",
285
+ 0xE01AA => "VS187",
286
+ 0xE01AB => "VS188",
287
+ 0xE01AC => "VS189",
288
+ 0xE01AD => "VS190",
289
+ 0xE01AE => "VS191",
290
+ 0xE01AF => "VS192",
291
+ 0xE01B0 => "VS193",
292
+ 0xE01B1 => "VS194",
293
+ 0xE01B2 => "VS195",
294
+ 0xE01B3 => "VS196",
295
+ 0xE01B4 => "VS197",
296
+ 0xE01B5 => "VS198",
297
+ 0xE01B6 => "VS199",
298
+ 0xE01B7 => "VS200",
299
+ 0xE01B8 => "VS201",
300
+ 0xE01B9 => "VS202",
301
+ 0xE01BA => "VS203",
302
+ 0xE01BB => "VS204",
303
+ 0xE01BC => "VS205",
304
+ 0xE01BD => "VS206",
305
+ 0xE01BE => "VS207",
306
+ 0xE01BF => "VS208",
307
+ 0xE01C0 => "VS209",
308
+ 0xE01C1 => "VS210",
309
+ 0xE01C2 => "VS211",
310
+ 0xE01C3 => "VS212",
311
+ 0xE01C4 => "VS213",
312
+ 0xE01C5 => "VS214",
313
+ 0xE01C6 => "VS215",
314
+ 0xE01C7 => "VS216",
315
+ 0xE01C8 => "VS217",
316
+ 0xE01C9 => "VS218",
317
+ 0xE01CA => "VS219",
318
+ 0xE01CB => "VS220",
319
+ 0xE01CC => "VS221",
320
+ 0xE01CD => "VS222",
321
+ 0xE01CE => "VS223",
322
+ 0xE01CF => "VS224",
323
+ 0xE01D0 => "VS225",
324
+ 0xE01D1 => "VS226",
325
+ 0xE01D2 => "VS227",
326
+ 0xE01D3 => "VS228",
327
+ 0xE01D4 => "VS229",
328
+ 0xE01D5 => "VS230",
329
+ 0xE01D6 => "VS231",
330
+ 0xE01D7 => "VS232",
331
+ 0xE01D8 => "VS233",
332
+ 0xE01D9 => "VS234",
333
+ 0xE01DA => "VS235",
334
+ 0xE01DB => "VS236",
335
+ 0xE01DC => "VS237",
336
+ 0xE01DD => "VS238",
337
+ 0xE01DE => "VS239",
338
+ 0xE01DF => "VS240",
339
+ 0xE01E0 => "VS241",
340
+ 0xE01E1 => "VS242",
341
+ 0xE01E2 => "VS243",
342
+ 0xE01E3 => "VS244",
343
+ 0xE01E4 => "VS245",
344
+ 0xE01E5 => "VS246",
345
+ 0xE01E6 => "VS247",
346
+ 0xE01E7 => "VS248",
347
+ 0xE01E8 => "VS249",
348
+ 0xE01E9 => "VS250",
349
+ 0xE01EA => "VS251",
350
+ 0xE01EB => "VS252",
351
+ 0xE01EC => "VS253",
352
+ 0xE01ED => "VS254",
353
+ 0xE01EE => "VS255",
354
+ 0xE01EF => "VS256",
318
355
  }.freeze
319
- COULD_BE_WHITESPACE = '[\p{Space}­᠎​‌‍⁠⁡⁢⁣⁤⠀𛲠𛲡𛲢𛲣𝅙𝅳𝅴𝅵𝅶𝅷𝅸𝅹𝅺]'.freeze
320
356
 
321
- def self.symbolify(char, encoding = char.encoding)
322
- return "n/a" if Unicode::Categories.category(char) == "Cn"
357
+ def self.symbolify(char, char_info)
358
+ if !char_info.valid?
359
+ "�"
360
+ else
361
+ case char_info
362
+ when UnicodeCharacteristics
363
+ Symbolify.unicode(char, char_info)
364
+ when ByteCharacteristics
365
+ Symbolify.byte(char, char_info)
366
+ when AsciiCharacteristics
367
+ Symbolify.ascii(char, char_info)
368
+ else
369
+ Symbolify.binary(char)
370
+ end
371
+ end
372
+ end
373
+
374
+ def self.unicode(char, char_info)
375
+ return "n/a" if !char_info.assigned?
323
376
 
324
377
  char = char.dup
378
+ ord = char.ord
379
+ encoding = char_info.encoding
325
380
 
326
- char.tr!(
327
- ASCII_CONTROL_CODEPOINTS.encode(encoding),
328
- ASCII_CONTROL_SYMBOLS.encode(encoding)
329
- )
330
- char.gsub!(
331
- Regexp.compile(COULD_BE_WHITESPACE.encode(encoding)),
332
- ']\0['.encode(encoding)
333
- )
381
+ if char_info.delete?
382
+ char = CONTROL_DELETE_SYMBOL
383
+ elsif char_info.c0?
384
+ char = CONTROL_C0_SYMBOLS[ord]
385
+ elsif char_info.c1?
386
+ char = CONTROL_C1_NAMES[ord]
387
+ elsif char_info.blank?
388
+ char = "]".encode(encoding) + char + "[".encode(encoding)
389
+ elsif ord > 917536 && ord < 917631
390
+ char = "TAG ".encode(encoding) +
391
+ char.tr(TAGS.encode(encoding), ASCII_CHARS.encode(encoding))
392
+ else
393
+ char = INTERESTING_CODEPOINTS[char.ord] || char
394
+ end
395
+
396
+ char.encode("UTF-8")
397
+ end
334
398
 
335
- INTERESTING_CODEPOINTS.each{ |cp, desc|
336
- char.gsub! Regexp.compile(cp.encode(encoding)), desc.encode(encoding)
337
- }
338
- char.gsub! TAG_START.encode(encoding), TAG_START_SYMBOL.encode(encoding)
339
- char.gsub! TAG_SPACE.encode(encoding), TAG_SPACE_SYMBOL.encode(encoding)
340
- char.gsub! TAG_DELETE.encode(encoding), TAG_DELETE_SYMBOL.encode(encoding)
399
+ def self.byte(char, char_info)
400
+ return "n/a" if !char_info.assigned?
341
401
 
342
402
  ord = char.ord
343
- if ord > 917536 && ord < 917631
344
- char.tr!(TAGS.encode(encoding), ASCII_CHARS.encode(encoding))
345
- char = "TAG ".encode(encoding) + char
403
+ encoding = char_info.encoding
404
+ no_converter = !!(NO_UTF8_CONVERTER =~ encoding.name)
405
+ treat_char_unconverted = false
406
+
407
+ if char_info.delete?
408
+ char = CONTROL_DELETE_SYMBOL
409
+ elsif char_info.c0?
410
+ char = CONTROL_C0_SYMBOLS[ord]
411
+ elsif char_info.c1?
412
+ char = CONTROL_C1_NAMES[ord]
413
+ elsif no_converter
414
+ treat_char_unconverted = true
415
+ elsif char_info.blank?
416
+ char = "]".encode(encoding) + char + "[".encode(encoding)
417
+ end
418
+
419
+ if no_converter && treat_char_unconverted
420
+ char.inspect
421
+ else
422
+ char.encode("UTF-8")
423
+ end
424
+ end
425
+
426
+ def self.ascii(char, char_info)
427
+ if char_info.delete?
428
+ char = CONTROL_DELETE_SYMBOL
429
+ elsif char_info.c0?
430
+ char = CONTROL_C0_SYMBOLS[char.ord]
431
+ elsif char_info.blank?
432
+ char = "]" + char + "["
346
433
  end
347
434
 
348
435
  char
349
436
  end
437
+
438
+ def self.binary(char)
439
+ char.inspect
440
+ end
350
441
  end
351
442
  end
data/lib/unibits/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Unibits
2
- VERSION = "1.3.0".freeze
2
+ VERSION = "2.0.0".freeze
3
3
  end
data/spec/unibits_spec.rb CHANGED
@@ -67,6 +67,24 @@ describe Unibits do
67
67
  result.must_match "01000011"
68
68
  end
69
69
 
70
+ it "works with 'ISO-8859-' encodings" do
71
+ string = "\xBC Idiosyncr\xE4tic\n\x91".force_encoding("ISO-8859-1")
72
+ result = Paint.unpaint(Unibits.visualize(string))
73
+ result.must_match "BC" # ¼
74
+ result.must_match "E4" # ä
75
+ result.must_match "␊" # \n
76
+ result.must_match "PU1" # C1 name for \x91
77
+ end
78
+
79
+ it "works with 'Windows-125' encodings" do
80
+ string = "\xBC Idiosyncr\xE4tic\n\x81".force_encoding("Windows-1252")
81
+ result = Paint.unpaint(Unibits.visualize(string))
82
+ result.must_match "BC" # ¼
83
+ result.must_match "E4" # ä
84
+ result.must_match "␊" # \n
85
+ result.must_match "n/a" # \x81 is not assigned
86
+ end
87
+
70
88
  describe "invalid UTF-8 encodings" do
71
89
  it "- unexpected continuation byte (1/2)" do
72
90
  string = "abc\x80efg"
data/unibits.gemspec CHANGED
@@ -19,7 +19,7 @@ Gem::Specification.new do |gem|
19
19
 
20
20
  gem.add_dependency 'paint', '>= 0.9', '< 3.0'
21
21
  gem.add_dependency 'unicode-display_width', '~> 1.1'
22
- gem.add_dependency 'unicode-categories', '~> 1.1', '>= 1.1.2'
22
+ gem.add_dependency 'characteristics', '~> 0.2.0'
23
23
  gem.add_dependency 'rationalist', '~> 2.0'
24
24
 
25
25
  gem.required_ruby_version = "~> 2.0"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: unibits
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 2.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jan Lelis
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-03-09 00:00:00.000000000 Z
11
+ date: 2017-03-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: paint
@@ -45,25 +45,19 @@ dependencies:
45
45
  - !ruby/object:Gem::Version
46
46
  version: '1.1'
47
47
  - !ruby/object:Gem::Dependency
48
- name: unicode-categories
48
+ name: characteristics
49
49
  requirement: !ruby/object:Gem::Requirement
50
50
  requirements:
51
51
  - - "~>"
52
52
  - !ruby/object:Gem::Version
53
- version: '1.1'
54
- - - ">="
55
- - !ruby/object:Gem::Version
56
- version: 1.1.2
53
+ version: 0.2.0
57
54
  type: :runtime
58
55
  prerelease: false
59
56
  version_requirements: !ruby/object:Gem::Requirement
60
57
  requirements:
61
58
  - - "~>"
62
59
  - !ruby/object:Gem::Version
63
- version: '1.1'
64
- - - ">="
65
- - !ruby/object:Gem::Version
66
- version: 1.1.2
60
+ version: 0.2.0
67
61
  - !ruby/object:Gem::Dependency
68
62
  name: rationalist
69
63
  requirement: !ruby/object:Gem::Requirement