terminal_rb 1.0.6 → 1.0.8

data/lib/terminal/text/formatter.rb

@@ -392,18 +392,41 @@ module Terminal
 
           line.encode!(ENCODING) if line.encoding != ENCODING
 
-          line.scan(SCAN_REGEX) do |newline, space, _, _, char|
-            if char
-              (last = @lex[-1]).is_a?(Word) and next last.add(char)
-              next @lex << Word.new(char)
+          line.scan(SCAN_REGEX) do |token|
+            b = token.getbyte(0)
+
+            if b > 0x20 # word chunk: an ASCII run or a single grapheme
+              # a single-byte token is printable ASCII: one character, one
+              # column - no need to ask for its width
+              size = token.bytesize
+              if size == 1
+                (last = @lex[-1]).is_a?(Word) and next last.add(token, 1)
+                next @lex << Word.new(token, 1)
+              end
+              # an ASCII run is the only multi-byte token which cannot be a
+              # grapheme cluster - there its width is simply its byte count
+              size = nil unless b < 0x80 && token.ascii_only?
+              if (last = @lex[-1]).is_a?(Word)
+                next size ? last.add(token, size, true) : last.add(token)
+              end
+              next @lex << Word.new(token, size, true) if size
+              next @lex << Word.new(token)
             end
 
-            if space
-              (last = @lex[-1]).is_a?(Space) and next last.inc
-              next @lex << Space.new
+            if SPACE_BYTE[b] && !(b == 0x0d && token.bytesize == 2)
+              size = token.bytesize
+              (last = @lex[-1]).is_a?(Space) and next last.size += size
+              next @lex << Space.new(size)
             end
 
-            next @lex << EOL if newline
+            next @lex << EOL if b == 0x0a || b == 0x0d
+
+            # escape sequences are dropped without `ansi`, any other control
+            # character is text
+            next if b == 0x1b && ((c = token.getbyte(1)) == 0x5b || c == 0x5d)
+
+            (last = @lex[-1]).is_a?(Word) and next last.add(token)
+            @lex << Word.new(token)
           end
 
           @lex[-1] == EOL ? @lex[-1] = EOP : @lex << EOP
@@ -421,23 +444,46 @@ module Terminal
           line.encode!(ENCODING) if line.encoding != ENCODING
           csis = 0
 
-          line.scan(SCAN_REGEX) do |newline, space, csi, osc, char|
-            if char
-              (last = @lex[-1]).is_a?(Word) and next last.add(char)
-              next @lex << Word.new(char)
+          line.scan(SCAN_REGEX) do |token|
+            b = token.getbyte(0)
+
+            if b > 0x20 # word chunk: an ASCII run or a single grapheme
+              # a single-byte token is printable ASCII: one character, one
+              # column - no need to ask for its width
+              size = token.bytesize
+              if size == 1
+                (last = @lex[-1]).is_a?(Word) and next last.add(token, 1)
+                next @lex << Word.new(token, 1)
+              end
+              # an ASCII run is the only multi-byte token which cannot be a
+              # grapheme cluster - there its width is simply its byte count
+              size = nil unless b < 0x80 && token.ascii_only?
+              if (last = @lex[-1]).is_a?(Word)
+                next size ? last.add(token, size, true) : last.add(token)
+              end
+              next @lex << Word.new(token, size, true) if size
+              next @lex << Word.new(token)
             end
 
-            if space
-              (last = @lex[-1]).is_a?(Space) and next last.inc
-              next @lex << Space.new
+            if SPACE_BYTE[b] && !(b == 0x0d && token.bytesize == 2)
+              size = token.bytesize
+              (last = @lex[-1]).is_a?(Space) and next last.size += size
+              next @lex << Space.new(size)
             end
 
-            next @lex << EOL if newline
+            next @lex << EOL if b == 0x0a || b == 0x0d
 
-            next @lex << Osc.new(osc) if osc
+            if (c = token.getbyte(1)) == 0x5d # Osc
+              next @lex << Osc.new(token)
+            end
+            if c != 0x5b # Csi
+              # a bare escape is just text
+              (last = @lex[-1]).is_a?(Word) and next last.add(token)
+              next @lex << Word.new(token)
+            end
 
-            # Handle csi...
-            if csi == "\e[m" || csi == "\e[0m"
+            # Handle Csi...
+            if token == "\e[m" || token == "\e[0m"
               next if csis == 0
 
               if @lex[-1].is_a?(Csi)
@@ -449,21 +495,17 @@ module Terminal
               next @lex << CsiEnd
             end
 
-            (last = @lex[-1]).is_a?(Csi) and next last.add(csi)
+            (last = @lex[-1]).is_a?(Csi) and next last.add(token)
 
             csis += 1
-            @lex << Csi.new(csi)
+            @lex << Csi.new(token)
           end
 
           @lex.pop if (last = @lex[-1]) == EOL
 
           next @lex << EOP if csis == 0
 
-          if last.is_a?(Csi) && csis == 1
-            @lex.pop
-          else
-            @lex << CsiEnd
-          end
+          last.is_a?(Csi) && csis == 1 ? @lex.pop : @lex << CsiEnd
           @lex << EOP
         end
 
@@ -517,33 +559,57 @@ module Terminal
       class Space
         attr_accessor :size
         def to_str = (@size == 1 ? ' ' : ' ' * @size)
-        def initialize = (@size = 1)
+        def initialize(size = 1) = (@size = size)
         def inc = (@size += 1)
         def inspect = @size == 1 ? '<Space>' : "<Space #{@size}>"
       end
 
       # @private
+      # A chunk is either a single grapheme cluster or a run of printable
+      # ASCII, for which the display width equals the character count.
       class Word
-        def to_str = (@to_str ||= @chars.join)
-        def size = (@size ||= @sizes.sum)
-        def inspect = "<Word #{@sizes.sum}:#{@chars.join.inspect}>"
-
-        def initialize(char, size = Text.char_width(char))
-          @chars = [char]
+        def to_str = (@to_str ||= @chunks.size == 1 ? @chunks[0] : @chunks.join)
+        def size = (@size ||= @sizes.size == 1 ? @sizes[0] : @sizes.sum)
+        def inspect = "<Word #{@sizes.sum}:#{@chunks.join.inspect}>"
+
+        # `run` marks a chunk holding more than one character; only the lexer
+        # creates those and it knows without having to inspect the chunk again
+        def initialize(chunk, size = Text.char_width(chunk), run = false)
+          @chunks = [chunk]
           @sizes = [size]
+          @runs = run
         end
 
-        def add(char, size = Text.char_width(char))
-          @chars << char
+        def add(chunk, size = Text.char_width(chunk), run = false)
+          @chunks << chunk
           @sizes << size
+          @runs ||= run
           nil
         end
 
         def split(width)
-          return [self] if @chars.size == 1
+          if @runs
+            # ASCII runs have to become single characters again to be split -
+            # a multi-character grapheme cluster must stay untouched
+            chars = []
+            sizes = []
+            @chunks.each_with_index do |chunk, idx|
+              if chunk.bytesize > 1 && chunk.getbyte(0) < 0x80 &&
+                   chunk.ascii_only?
+                chunk.each_char { chars << it and sizes << 1 }
+              else
+                chars << chunk
+                sizes << @sizes[idx]
+              end
+            end
+          else
+            chars = @chunks
+            sizes = @sizes
+          end
+          return [self] if chars.size == 1
           ret = last = size = nil
-          @chars.each_with_index do |char, idx|
-            csize = @sizes[idx]
+          chars.each_with_index do |char, idx|
+            csize = sizes[idx]
             next ret = [last = Word.new(char, size = csize)] unless ret
             next last.add(char, csize) if (size += csize) <= width
             ret << (last = Word.new(char, size = csize))
@@ -597,16 +663,36 @@ module Terminal
       space_cache[2]
       SPACE_CACHE = space_cache.freeze
 
+      # A run of printable ASCII is only taken when the following character is
+      # ASCII too: everything which can extend a grapheme cluster (combining
+      # marks, ZWJ, variation selectors, ...) is non-ASCII, so a run can never
+      # swallow the base of a cluster `\X` would have kept together.
       SCAN_REGEX =
-        /\G(?:
-         (\r?\n)
-        | (\s)
-        | (\e\[[\x30-\x3f]*[\x20-\x2f]*[a-zA-Z])
-        | (\e\]\d+(?:;[^\a\e]+)*(?:\a|\e\\))
-        | (\X)
-      )/x
+        /
+          [\x21-\x7e]+(?=[\x00-\x7f]|\z)
+        | \x20+
+        | \r?\n
+        | \e\[[\x30-\x3f]*[\x20-\x2f]*[a-zA-Z]
+        | \e\]\d+(?:;[^\a\e]+)*(?:\a|\e\\)
+        | \s
+        | \X
+      /x
+
+      SPACE_BYTE =
+        Hash
+          .new(false)
+          .compare_by_identity
+          .merge!(
+            0x20 => true,
+            0x09 => true,
+            0x0b => true,
+            0x0c => true,
+            0x0d => true
+          )
+          .freeze
 
       private_constant(
+        :SPACE_BYTE,
         :Space,
         :Word,
         :Osc,

data/lib/terminal/text.rb

@@ -29,7 +29,13 @@ module Terminal
       #   Terminal::Text.ambiguous_char_width = 2 # for CJK terminals
       #
       # @return [Integer] default: `1`
-      attr_accessor :ambiguous_char_width
+      attr_reader :ambiguous_char_width
+
+      # @attribute [w] ambiguous_char_width
+      def ambiguous_char_width=(value)
+        autoload?(:CharWidth) or CharWidth.clear
+        @ambiguous_char_width = value
+      end
 
       # Calculate the display width of a string in terminal columns.
       #
@@ -206,66 +212,66 @@ module Terminal
       def char_width(char)
         ord = char.ord
         return @ctrlchar_width[ord] if ord < 0x20
-        if char.size == 1
-          return ord < 0xa1 ? 1 : CharWidth[ord] || @ambiguous_char_width
-        end
+        # CharWidth knows this rule too - a comparison just beats a memo hit
+        return ord < 0xa1 ? 1 : CharWidth[ord] if char.bytesize == 1
+        return 2 if ord == 0x1f3f3 && @very_special_flags.include?(char)
         sum = 0
-        zwj = false
-        char.each_char do |c|
-          next zwj = false if zwj
-          ord = c.ord
-          next zwj = true if ord == 0x200d # zero with joiner
-          sum += CharWidth[ord] || @ambiguous_char_width
+        skip = false
+        char.each_codepoint do |cp|
+          next skip = false if skip
+          next skip = true if cp == 0x200d # zero width joiner
+          sum += CharWidth[cp]
         end
         sum
       end
     end
 
-    @ambiguous_char_width = 1
+    @very_special_flags = [
+      "🏳\u{FE0F}\u{200D}🌈", # rainbow flag
+      "🏳\u{FE0F}\u{200D}⚧\u{FE0F}" # transgender flag
+    ].freeze
 
-    @ctrlchar_width =
-      Hash
-        .new(2)
-        .compare_by_identity
-        .merge!(
-          0x00 => 0,
-          0x01 => 1,
-          0x02 => 1,
-          0x03 => 1,
-          0x04 => 1,
-          0x05 => 0,
-          0x06 => 1,
-          0x07 => 0,
-          0x08 => 0,
-          0x09 => 8,
-          0x0a => 0,
-          0x0b => 0,
-          0x0c => 0,
-          0x0d => 0,
-          0x0e => 0,
-          0x0f => 0,
-          0x10 => 1,
-          0x11 => 1,
-          0x12 => 1,
-          0x13 => 1,
-          0x14 => 1,
-          0x15 => 1,
-          0x16 => 1,
-          0x17 => 1,
-          0x18 => 1,
-          0x19 => 1,
-          0x1a => 1,
-          0x1b => 1,
-          0x1c => 1,
-          0x1d => 1,
-          0x1e => 1,
-          0x1f => 1
-        )
-        .freeze
+    # indexed by ordinal, only ever asked for control characters
+    @ctrlchar_width = [
+      0,
+      1,
+      1,
+      1,
+      1,
+      0,
+      1,
+      0,
+      0,
+      1,
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1,
+      1
+    ].freeze
+
+    @ambiguous_char_width = 1
 
-    cw_file = "#{__dir__}/text/char_width.rb"
-    autoload :CharWidth, cw_file
-    autoload :UNICODE_VERSION, cw_file
+    char_width_file = "#{__dir__}/text/char_width.rb"
+    autoload :CharWidth, char_width_file
+    autoload :UNICODE_VERSION, char_width_file
     private_constant :CharWidth
   end
 end

data/lib/terminal/version.rb

@@ -2,5 +2,5 @@
 
 module Terminal
   # The version number of the gem.
-  VERSION = '1.0.6'
+  VERSION = '1.0.8'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: terminal_rb
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.8
 platform: ruby
 authors:
 - Mike Blumtritt
@@ -60,7 +60,7 @@ metadata:
   yard.run: yard
   source_code_uri: https://codeberg.org/mblumtritt/Terminal.rb
   bug_tracker_uri: https://codeberg.org/mblumtritt/Terminal.rb/issues
-  documentation_uri: https://rubydoc.info/gems/terminal_rb/1.0.6
+  documentation_uri: https://rubydoc.info/gems/terminal_rb/1.0.8
 rdoc_options: []
 require_paths:
 - lib
@@ -75,7 +75,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.15
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Fast terminal access with ANSI, CSIu, mouse events, BBCode, word-wise line
   break support and much more.