lzutf8 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8cd5f3fa2daec203af83a69d6e4ec405365755e3c6ce5065e089a4ad97dbc49b
+  data.tar.gz: a1b0046f6df140c6a19ed723f57317eb370706168eaba396d04d988fa5a23eac
+SHA512:
+  metadata.gz: 816c9a697db4a67e4c8216a84a1841598c7b7898991b8a9dbe80c13d9854a34b7f9e3636b393da09d8c7b6a560c870dc997685e00ad78333679537de15b08e46
+  data.tar.gz: c6bc288cf364ceecc14365bf5c3c9308c26341902fc583dfb8af83ed83fea9c1c778591d2609f6176a265857ad1dd469cd2ebc1b313d786d896037aa62dc806f

data/.yardopts

@@ -0,0 +1,3 @@
+-M redcarpet -m markdown
+
+lib/lzutf8.rb lib/**/*.rb

data/README.md

@@ -0,0 +1,47 @@
+# LZUTF8 Gem - providing `lzutf8`
+[![Gem Version](https://badge.fury.io/rb/lzutf8.svg)](https://rubygems.org/gems/lzutf8)
+[![Documentation](http://img.shields.io/badge/docs-rdoc.info-blue.svg)](http://www.rubydoc.info/gems/lzutf8/0.0.1)
+A ruby gem containing an implementation of LZUTF-8 compression and decompression
+
+* `LZUTF8.compress("some input")`
+* `LZUTF8.decompress("some compressed input")`
+
+## Algorithm
+
+This is a port of https://github.com/rotemdan/lzutf8.js/
+
+The trick is to create a set of pointer sequences that coexist with UTF-8 encoding.  The pointers include fields for a matched sequence length (bits indicated with `l`) and a distance (backward) that the sequence can be found in the encoded string (bits indicated with `d`).
+
+|Bytes |Sized pointer sequence      | UTF-8 Codepoint sequence
+|------|----------------------------|-------------------------
+| 1    |n/a                         |`0xxxxxxx`
+| 2    |`110lllll 0ddddddd`         |`110xxxxx 10xxxxxx`
+| 3    |`111lllll 0ddddddd dddddddd`|`1110xxxx 10xxxxxx 10xxxxxx`
+| 4    |n/a                         |`11110xxx 10xxxxxx 10xxxxxx 10xxxxxx`
+
+Note that this allows for 5 bits (representing maxiumum 31 bytes) of matched sequence and up to 15 bits (representing maximum 32767 bytes) of distance to the matched sequence.
+
+### Compressing
+
+The text is converted to bytes of UTF-8.  Byte sequences of `4 <= length` and `length <= 31` are replaced where possible with sized pointer sequences -- pointing to a relative location up to 32767 bytes where the sequence can be found.
+
+Hash every 4-byte sequence in the input string and use it to store the position of that sequence.  Each hash bucket will then contain an array of locations where that starting position can be found.
+
+For example, the string `abcdefabcd` would produce the following table:
+
+|hash  |pointers
+|------|-------
+|`abcd`|0, 6
+|`bcde`|1
+|`cdef`|2
+|`defa`|3
+|`efab`|4
+|`fabc`|5
+
+The output string would be identical to the input as far as `abcdef` after which a sized pointer representing a distance of 6 and a length of 4 would be appended: `0b11000100_00000110`
+
+### Decompressing
+
+Scan for sized pointers based on the bit sequences listed above.  When encountered, replace them with the text at the pointed-to location of the desired length.
+
+> **Note:** it is legal for the desired length to be _longer_ than the requested distance (e.g. a repeated individual character might produce a sized pointer that requests length 31 from distance 1).  The text extracted from the pointer should be repeated to fill the desired length.

data/lib/lzutf8.rb

@@ -0,0 +1,208 @@
+require "lzutf8/version"
+
+# LZUTF8 contains functions to compress and decompress UTF-8 text with the LZUTF-8 algorithm
+# @author Ian Katz <ianfixes@gmail.com>
+module LZUTF8
+
+  MINIMUM_SEQUENCE_LENGTH = 4
+  MAXIMUM_SEQUENCE_LENGTH = 31
+  MAXIMUM_MATCH_DISTANCE = 32767
+
+  # Extract the information contained in an LZUTF8 Sized Pointer
+  #
+  # @param bytes [Array<Integer>] an array of character codes
+  # @return [Integer, Integer] the pointer length and distance
+  def self.pointer_info(bytes)
+    c1, c2, c3 = bytes
+    length = c1 & 0b00011111
+    # either llllll_0ddddddd or 0b111lllll_0ddddddd_dddddddd
+    distance = if (c1 & 0b00100000).zero?
+      c2
+    else
+      (c2 << 8) + c3
+    end
+    [length, distance]
+  end
+
+  # Explain a compressed (or uncompressed) UTF-8 string at the sequence-of-bits level by printing to STDOUT
+  #
+  # @param string [String] the input string
+  def self.explain(string)
+    raise ArgumentError unless string.is_a? String
+
+    input = string.bytes
+    position = -1
+    outsize = 0
+
+    pointer2 = lambda do |seq|
+      c1, c2 = seq
+      [c2].none?(&:nil?) &&
+        (c1 & 0b11100000) == 0b11000000 &&
+        (c2 & 0b10000000).zero?
+    end
+
+    pointer3 = lambda do |seq|
+      c1, c2, c3 = seq
+      [c2, c3].none?(&:nil?) &&
+        (c1 & 0b11100000) == 0b11100000 &&
+        (c2 & 0b10000000).zero?
+    end
+
+    codepoint1 = ->(seq) { (seq.first & 0b10000000).zero? }
+
+    codepoint2 = lambda do |seq|
+      c1, c2 = seq
+      [c2].none?(&:nil?) &&
+        (c1 & 0b11100000) == 0b11000000 &&
+        (c2 & 0b11000000) == 0b10000000
+    end
+
+    codepoint3 = lambda do |seq|
+      c1, c2, c3 = seq
+      [c2, c3].none?(&:nil?) &&
+        (c1 & 0b11110000) == 0b11100000 &&
+        (c2 & 0b11000000) == 0b10000000 &&
+        (c3 & 0b11000000) == 0b10000000
+    end
+
+    codepoint4 = lambda do |seq|
+      c1, c2, c3, c4 = seq
+      [c2, c3, c4].none?(&:nil?) &&
+        (c1 & 0b11111000) == 0b11110000 &&
+        (c2 & 0b11000000) == 0b10000000 &&
+        (c3 & 0b11000000) == 0b10000000 &&
+        (c4 & 0b11000000) == 0b10000000
+    end
+
+    binarize = proc { |bytes| "0b" + bytes.map { |b| b.to_s(2).rjust(8, '0') }.join("_") }
+    dump = proc do |pos, bytes, inc, meaning|
+      outsize += inc
+      puts "#{pos.to_s.rjust(4, '0')} #{binarize.call(bytes)} #{meaning} OS=#{outsize}"
+    end
+
+    dumpc = proc { |pos, bytes| dump.call(pos, bytes, bytes.size, "literal - #{bytes.pack('C*')}") }
+    dumpp = proc do |pos, bytes|
+      length, distance = self.pointer_info(bytes)
+      dump.call(pos, bytes, length, "pointer l=#{length} d=#{distance}")
+    end
+
+    until input.empty? do
+      position += 1
+      c1 = input.shift
+      c2, c3, c4 = input[0, 3]
+      sequence = [c1, c2, c3, c4]
+
+      case sequence
+      when codepoint1
+        dumpc.call(position, [c1])
+      when codepoint2
+        dumpc.call(position, [c1, c2])
+        position += 1.times { input.shift } # rubocop:disable Lint/UselessTimes
+      when codepoint3
+        dumpc.call(position, [c1, c2, c3])
+        position += 2.times { input.shift }
+      when codepoint4
+        dumpc.call(position, [c1, c2, c3, c4])
+        position += 3.times { input.shift }
+      when pointer2
+        dumpp.call(position, [c1, c2])
+        position += 1.times { input.shift } # rubocop:disable Lint/UselessTimes
+      when pointer3
+        dumpp.call(position, [c1, c2, c3])
+        position += 2.times { input.shift }
+      else
+        dump.call(position, [c1], 1, "Assumed part of a sequence")
+      end
+    end
+  end
+
+  # Decompress an LZUTF8-compressed string.
+  #
+  # Due to the nature of this decompression algorithm, any uncompressed UTF-8 string can be
+  # passed to this function and will be returned unmodified
+  #
+  # @param string [String] The input LZUTF8-compressed string
+  # @return [String] Uncompressed UTF-8 string
+  def self.decompress(string)
+    raise ArgumentError unless string.is_a? String
+
+    input = string.bytes
+    output = []
+    until input.empty? do
+      c1 = input.shift # consume
+      c2 = input.first # peek
+      next (output << c1) unless (c1 & 0b11000000) == 0b11000000 && (c2 & 0b10000000).zero?
+
+      # By this point we know it's not a literal char and we must actually consume the 2nd byte
+      # either llllll_0ddddddd or 0b111lllll_0ddddddd_dddddddd
+      c2       = input.shift & 0b01111111
+      length   = c1 & 0b00011111
+      distance = (c1 & 0b00100000).zero? ? c2 : (c2 << 8) + input.shift # consume 3rd byte if needed
+
+      # get text from pointer, wrap in enumartor, take data until length satisfied and append
+      output += Enumerator.new { |y| loop { output[-distance, length].each { |v| y << v } } }.lazy.take(length).to_a
+    end
+
+    output.pack('C*').force_encoding(Encoding::UTF_8)
+  end
+
+  # Decompress a string using the LZUTF8 algorithm
+  #
+  # Due to the nature of this compression algorithm, only valid UTF-8 codepoints can be compressed;
+  # arbitrary binary data will fail.
+  #
+  # @param string [String] The input string
+  # @return [String] compresed string
+  def self.compress(string)
+    raise ArgumentError unless string.is_a? String
+
+    input = string.bytes
+    hash = {}
+    match_score = proc { |dist, len| dist < 128 ? len * 1.5 : len } # 2 byte vs 3 byte compression
+
+    pointer = -1
+    output = []
+    until pointer + 1 == input.size do
+      pointer += 1
+      c1, c2, c3, c4 = key = input[pointer, 4]
+      key = key.pack('C*')
+      next (output << c1) if [c2, c3, c4].any?(&:nil?) # near end of input, just iterate until it's consumed
+
+      max_len = [input.size - pointer, MAXIMUM_SEQUENCE_LENGTH].min  # max length of a match
+      matches = begin
+        next (output << c1) if hash[key].nil? # no matches if no bucket
+
+        hash[key].map do |start|              # all known bucket entries as [distance, length_of_match]
+          matchable = input[pointer, max_len] # max length of the matchable input segment
+          distance = pointer - start          # relative distance must be less than max expressable
+          next nil if distance > MAXIMUM_MATCH_DISTANCE # this would mean a bucket entry we didn't clean up
+
+          # linear comparison to find longest common prefix
+          len = input[start, max_len].zip(matchable).index { |a, b| a.nil? || b.nil? || a != b }
+          case len
+          when nil then                   [distance, matchable.size] # hit end of input, fully matched
+          when 0..MINIMUM_SEQUENCE_LENGTH then nil                   # no match
+          else                            [distance, len - 1]        # index is the one BEFORE the mismatch
+          end
+        end
+      ensure  # that the pointer is added to the hash for this sequence
+        hash[key] = [] if hash[key].nil?
+        hash[key] << pointer
+        hash[key] = hash[key].select { |p| pointer - p < MAXIMUM_MATCH_DISTANCE } # prune too-distant matches
+      end.compact
+
+      best_match = matches.max { |a, b| match_score.call(a) <=> match_score.call(b) }
+      next (output << c1) if best_match.nil?
+
+      # Output a pointer to the match and advance the input pointer by the amount matched
+      match_distance, match_length = best_match
+      output += if match_distance < 0b10000000
+        [0b11000000 | match_length, match_distance]
+      else
+        [0b11100000 | match_length, match_distance >> 8, match_distance & 0b00000000_11111111]
+      end
+      pointer += (match_length - 1)
+    end
+    output.pack('C*').force_encoding(Encoding::UTF_8)
+  end
+end

data/lib/lzutf8/version.rb

@@ -0,0 +1,3 @@
+module LZUTF8
+  VERSION = "0.0.1".freeze
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: lzutf8
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ian Katz
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2021-03-01 00:00:00.000000000 Z
+dependencies: []
+description: ''
+email:
+- ianfixes@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".yardopts"
+- README.md
+- lib/lzutf8.rb
+- lib/lzutf8/version.rb
+homepage: http://github.com/ianfixes/lzutf8_gem
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key:
+specification_version: 4
+summary: Compression and decompression implementation of LZUTF-8 Algorithm
+test_files: []