scrapy_rocket_lz_string 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a95b8862aed6ff437dac452d62a46561131c23c76da700e66fa48d2525006f13
+  data.tar.gz: 43d2b1b63d77d521d0b2ff7b648ea59adaa8ece5b095bec08ff083d9e48953ff
+SHA512:
+  metadata.gz: 658bc68aecc7e4569d0c3fb3a431059d76c9d07065b4041c3e7b717fbfed5fb01000bd6f2366182b359d563b77440682fa5523fb9faa956166d3dc3a8be42aec
+  data.tar.gz: 7ffdf7dd01b2b7eb0d65706a9fc3e1ca36cb18e2d2b13794913dd383e4d23d080cd0c29ad2fd4a83b3751cd830c766f210ee4b804b4d7a8c151e42c2e410ab92

data/README.md

@@ -0,0 +1,67 @@
+# lz-string
+Ruby implementation of [LZ-String](https://github.com/pieroxy/lz-string) compression algorithm.
+
+Supports:
+ * Raw compression
+ * UTF-16 compression
+ * Base64 compression
+
+### Installation
+
+Install the latest release:
+
+```
+$ gem install lz_string
+```
+
+In Rails, add it to your Gemfile:
+
+```ruby
+gem 'lz_string'
+```
+
+### How to use
+
+#### Normal Compression and Decompression:
+
+``` ruby
+  # Compress
+  compressed = LZString.compress("Hello world!")
+  => "҅〶惶@✰Ӏ葀"
+  
+  # Decompress
+  LZString.decompress(compressed)
+  => "Hello world!"
+```
+
+#### UTF-16 Compression and Decompression:
+
+``` ruby
+  some_string = '{"some": "json", "foo": [{"bar": "؋", "key": "؄"}], "ঞᕠ": "൱ඵቜ"}'
+  
+  # Compress
+  compressed = LZString::UTF16.compress(some_string)
+  => "ᯡࡓ䈌\u0B80匰ᜠр\u0AF2Ǹ䀺㈦イ\u0530්C¦¼䒨ᨬිnj〩痐࠸С㸢璑Ч䲤U⋴ҕ䈥㛢ĉ႙  "
+  
+  # Decompress
+  LZString::UTF16.decompress(compressed)
+  => "{\"some\": \"json\", \"foo\": [{\"bar\": \"؋\", \"key\": \"؄\"}], \"ঞᕠ\": \"൱ඵቜ\"}"
+```
+
+#### Base64 Compression and Decompression:
+
+``` ruby		
+  # Compress
+  compressed = LZString::Base64.compress("Hello world!")
+  => "BIUwNmD2AEDukCcwBMCEQ==="
+  
+  # Decompress
+  LZString::Base64.decompress(compressed)
+  => "Hello world!"
+```
+ 
+### Tests
+
+``` bash
+$ rake
+```

data/lib/lz_string/base.rb

@@ -0,0 +1,397 @@
+module LZString
+  # Base compression class.
+  class Base
+    # @param uncompressed      [String]
+    # @param bits_per_char     [Integer]
+    # @param get_char_from_int [Integer]
+    def self.compress(uncompressed, bits_per_char, get_char_from_int)
+      return "" if uncompressed.nil?
+
+      i, value, ii = nil
+      context_dictionary = {}
+      context_dictionary_to_create = {}
+      context_c = ""
+      context_wc = ""
+      context_w = ""
+      # Compensate for the first entry which should not count
+      context_enlarge_in = 2
+      context_dict_size = 3
+      context_num_bits = 2
+      context_data = []
+      context_data_val = 0
+      context_data_position = 0
+
+      for ii in 0...uncompressed.length do
+        context_c = uncompressed[ii]
+
+        if (!context_dictionary.has_key?(context_c))
+          context_dictionary[context_c] = context_dict_size
+          context_dictionary_to_create[context_c] = true
+          context_dict_size += 1
+        end
+
+        context_wc = context_w + context_c
+        if (context_dictionary.has_key?(context_wc))
+          context_w = context_wc
+        else
+          if (context_dictionary_to_create.has_key?(context_w))
+            if (context_w[0].ord < 256)
+              for i in 0...context_num_bits do
+                context_data_val = (context_data_val << 1)
+                if (context_data_position == bits_per_char - 1)
+                  context_data_position = 0
+                  context_data.push(get_char_from_int[context_data_val])
+                  context_data_val = 0
+                else
+                  context_data_position += 1
+                end
+              end
+
+              value = context_w[0].ord
+              for i in 0...8 do
+                context_data_val = (context_data_val << 1) | (value & 1)
+                if (context_data_position == bits_per_char - 1)
+                  context_data_position = 0
+                  context_data.push(get_char_from_int[context_data_val])
+                  context_data_val = 0
+                else
+                  context_data_position += 1
+                end
+                value = value >> 1
+              end
+            else
+              value = 1
+              for i in 0...context_num_bits do
+                context_data_val = (context_data_val << 1) | value
+                if (context_data_position == bits_per_char-1)
+                  context_data_position = 0
+                  context_data.push(get_char_from_int[context_data_val])
+                  context_data_val = 0
+                else
+                  context_data_position += 1
+                end
+                value = 0
+              end
+              value = context_w[0].ord
+              for i in 0...16 do
+                context_data_val = (context_data_val << 1) | (value & 1)
+                if (context_data_position == bits_per_char - 1)
+                  context_data_position = 0
+                  context_data.push(get_char_from_int[context_data_val])
+                  context_data_val = 0
+                else
+                  context_data_position += 1
+                end
+                value = value >> 1
+              end
+            end
+            context_enlarge_in -= 1
+            if (context_enlarge_in == 0)
+              context_enlarge_in = 2**context_num_bits
+              context_num_bits += 1
+            end
+            context_dictionary_to_create.delete(context_w)
+          else
+            value = context_dictionary[context_w]
+            for i in 0...context_num_bits do
+              context_data_val = (context_data_val << 1) | (value & 1)
+              if (context_data_position == bits_per_char - 1)
+                context_data_position = 0
+                context_data.push(get_char_from_int[context_data_val])
+                context_data_val = 0
+              else
+                context_data_position += 1
+              end
+              value = value >> 1
+            end
+          end
+
+          context_enlarge_in -= 1
+
+          if (context_enlarge_in == 0)
+            context_enlarge_in = 2**context_num_bits
+            context_num_bits += 1
+          end
+
+          # Add wc to the dictionary.
+          context_dictionary[context_wc] = context_dict_size
+          context_dict_size += 1
+          context_w = context_c.to_s
+        end
+      end
+
+      # Output the code for w.
+      if (context_w != "")
+        if (context_dictionary_to_create.has_key?(context_w))
+          if (context_w[0].ord < 256)
+            for i in 0...context_num_bits do
+              context_data_val = (context_data_val << 1)
+              if (context_data_position == bits_per_char-1)
+                context_data_position = 0
+                context_data.push(get_char_from_int[context_data_val])
+                context_data_val = 0
+              else
+                context_data_position += 1
+              end
+            end
+            value = context_w[0].ord
+            for i in 0...8 do
+              context_data_val = (context_data_val << 1) | (value & 1)
+              if (context_data_position == bits_per_char-1)
+                context_data_position = 0
+                context_data.push(get_char_from_int[context_data_val])
+                context_data_val = 0
+              else
+                context_data_position += 1
+              end
+              value = value >> 1
+            end
+          else
+            value = 1
+            for i in 0...context_num_bits do
+              context_data_val = (context_data_val << 1) | value
+              if (context_data_position == bits_per_char-1)
+                context_data_position = 0
+                context_data.push(get_char_from_int[context_data_val])
+                context_data_val = 0
+              else
+                context_data_position += 1
+              end
+              value = 0
+            end
+            value = context_w[0].ord
+            for i in 0...16 do
+              context_data_val = (context_data_val << 1) | (value & 1)
+              if (context_data_position == bits_per_char-1)
+                context_data_position = 0
+                context_data.push(get_char_from_int[context_data_val])
+                context_data_val = 0
+              else
+                context_data_position += 1
+              end
+              value = value >> 1
+            end
+          end
+          context_enlarge_in -= 1
+          if (context_enlarge_in == 0)
+            context_enlarge_in = 2**context_num_bits
+            context_num_bits += 1
+          end
+          context_dictionary_to_create.delete(context_w)
+        else
+          value = context_dictionary[context_w]
+          for i in 0...context_num_bits
+            context_data_val = (context_data_val << 1) | (value & 1)
+            if (context_data_position == bits_per_char-1)
+              context_data_position = 0
+              context_data.push(get_char_from_int[context_data_val])
+              context_data_val = 0
+            else
+              context_data_position += 1
+            end
+            value = value >> 1
+          end
+        end
+
+        context_enlarge_in -= 1
+        if (context_enlarge_in == 0)
+          context_enlarge_in = 2**context_num_bits
+          context_num_bits += 1
+        end
+      end
+
+      # Mark the end of the stream
+      value = 2
+      for i in 0...context_num_bits do
+        context_data_val = (context_data_val << 1) | (value & 1)
+        if (context_data_position == bits_per_char - 1)
+          context_data_position = 0
+          context_data.push(get_char_from_int[context_data_val])
+          context_data_val = 0
+        else
+          context_data_position += 1
+        end
+        value = value >> 1
+      end
+
+      # Flush the last char
+      while (true)
+        context_data_val = (context_data_val << 1)
+        if (context_data_position == bits_per_char-1)
+          context_data.push(get_char_from_int[context_data_val])
+          break
+        else
+          context_data_position += 1
+        end
+      end
+
+      return context_data.join("")
+    end
+
+    # @param length         [Integer]
+    # @param reset_value    [Integer]
+    # @param get_next_value [Proc]
+    def self.decompress(length, reset_value, get_next_value, encoding = "ASCII-8BIT")
+      dictionary = [0, 1, 2]
+      enlarge_in = 4
+      dict_size = 4
+      num_bits = 3
+      entry = ""
+      result = []
+      data = {
+        val: get_next_value[(0)],
+        position: reset_value,
+        index: 1
+      }
+      bits = 0
+      maxpower = 2**2
+      power = 1
+      i, w, resb, c = nil
+
+      while (power != maxpower)
+        resb = data[:val] & data[:position]
+        data[:position] >>= 1
+        if (data[:position] == 0)
+          data[:position] = reset_value
+          data[:val] = get_next_value[data[:index]]
+          data[:index] += 1
+        end
+        bits |= (resb > 0 ? 1 : 0) * power
+        power <<= 1
+      end
+
+      case(n = bits)
+      when 0
+        bits = 0
+        maxpower = 2**8
+        power = 1
+        while power != maxpower
+          resb = data[:val] & data[:position]
+          data[:position] >>= 1
+          if (data[:position] == 0)
+            data[:position] = reset_value
+            data[:val] = get_next_value[data[:index]]
+            data[:index] += 1
+          end
+          bits |= (resb > 0 ? 1 : 0) * power
+          power <<= 1
+        end
+        c = bits.chr(encoding)
+      when 1
+        bits = 0
+        maxpower = 2*16
+        power = 1
+        while (power != maxpower)
+          resb = data[:val] & data[:position]
+          data[:position] >>= 1
+          if (data[:position] == 0)
+              data[:position] = reset_value
+              data[:val] = get_next_value[data[:index]]
+              data[:index] += 1
+          end
+          bits |= (resb > 0 ? 1 : 0) * power
+          power <<= 1
+        end
+        c = bits.chr(encoding)
+      when 2
+        ""
+      end
+
+      dictionary[3] = c
+      w = c
+      result << c
+
+      while(true)
+        return "" if (data[:index] > length)
+
+        bits = 0
+        maxpower = 2**num_bits
+        power = 1
+        while (power != maxpower)
+          resb = data[:val] & data[:position]
+          data[:position] >>= 1
+          if (data[:position] == 0)
+            data[:position] = reset_value
+            data[:val] = get_next_value[data[:index]]
+            data[:index] += 1
+          end
+          bits |= (resb > 0 ? 1 : 0) * power
+          power <<= 1
+        end
+
+        case(c = bits)
+        when 0
+          bits = 0
+          maxpower = 2**8
+          power = 1
+          while (power != maxpower)
+            resb = data[:val] & data[:position]
+            data[:position] >>= 1
+            if (data[:position] == 0)
+              data[:position] = reset_value
+              data[:val] = get_next_value[data[:index]]
+              data[:index] += 1
+            end
+            bits |= (resb > 0 ? 1 : 0) * power
+            power <<= 1
+          end
+
+          dictionary[dict_size] = bits.chr(encoding)
+          dict_size += 1
+          c = dict_size - 1
+          enlarge_in -= 1
+        when 1
+          bits = 0
+          maxpower = 2**16
+          power = 1
+          while (power != maxpower)
+            resb = data[:val] & data[:position]
+            data[:position] >>= 1
+            if (data[:position] == 0)
+              data[:position] = reset_value
+              data[:val] = get_next_value[data[:index]]
+              data[:index] += 1
+            end
+            bits |= (resb > 0 ? 1 : 0) * power
+            power <<= 1
+          end
+          dictionary[dict_size] = bits.chr(encoding)
+          dict_size += 1
+          c = dict_size - 1
+          enlarge_in -= 1
+        when 2
+          return result.join("")
+        end
+
+        if (enlarge_in == 0)
+          enlarge_in = 2**num_bits
+          num_bits += 1
+        end
+
+        if (dictionary[c])
+          entry = dictionary[c]
+        else
+          if (c === dict_size)
+              entry = w + w[0]
+          else
+            return nil
+          end
+        end
+
+        result << entry
+
+        # Add w+entry[0] to the dictionary.
+        dictionary[dict_size] = w + entry[0]
+        dict_size += 1
+        enlarge_in -= 1
+
+        w = entry
+
+        if (enlarge_in == 0)
+          enlarge_in = 2**num_bits
+          num_bits += 1
+        end
+      end
+    end
+  end
+end

data/lib/lz_string/base64.rb

@@ -0,0 +1,51 @@
+module LZString
+  # Base64 compressing algorithm.
+  class Base64
+    # Base64 alphabet.
+    KEY_STR_BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
+
+    # @param input [String]
+    def self.compress(input)
+      return "" if input.nil?
+
+      res = LZString::Base.compress(input, 6, lambda { |a| KEY_STR_BASE64[a] })
+
+      # To produce valid Base64
+      case (res.length % 4)
+      when 0 then res
+      when 1 then res + "==="
+      when 2 then res + "=="
+      when 3 then res + "="
+      end
+    end
+
+    # @param compressed [String]
+    def self.decompress(compressed)
+      return "" if compressed.nil?
+      return nil if compressed == ""
+
+      LZString::Base.decompress(
+        compressed.length,
+        32,
+        lambda { |index| get_base_value(KEY_STR_BASE64, compressed[index]) }
+      )
+    end
+
+    # @param alphabet  [String]
+    # @param character [String]
+    def self.get_base_value(alphabet, character)
+      base_reverse_dic = {}
+
+      if (!base_reverse_dic[alphabet])
+        base_reverse_dic[alphabet] = {}
+        for i in 0...alphabet.length do
+          base_reverse_dic[alphabet][alphabet[i]] = i
+        end
+      end
+
+      base_reverse_dic[alphabet][character]
+    end
+
+    private_class_method :get_base_value
+  end
+end

data/lib/lz_string/urisafe.rb

@@ -0,0 +1,42 @@
+module LZString
+  class UriSafe
+    # Base64 alphabet.
+    KEY_STR_URISAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-$"
+
+    # @param input [String]
+    def self.compress(input)
+      return "" if input.nil?
+
+      LZString::Base.compress(input, 6, lambda { |a| KEY_STR_URISAFE[a] })
+    end
+
+    # @param compressed [String]
+    def self.decompress(compressed)
+      return "" if compressed.nil?
+      return nil if compressed == ""
+      compressed.gsub!(" ","+")
+      LZString::Base.decompress(
+        compressed.length,
+        32,
+        lambda { |index| get_base_value(KEY_STR_URISAFE, compressed[index]) }
+      )
+    end
+
+    # @param alphabet  [String]
+    # @param character [String]
+    def self.get_base_value(alphabet, character)
+      base_reverse_dic = {}
+
+      if (!base_reverse_dic[alphabet])
+        base_reverse_dic[alphabet] = {}
+        for i in 0...alphabet.length do
+          base_reverse_dic[alphabet][alphabet[i]] = i
+        end
+      end
+
+      base_reverse_dic[alphabet][character]
+    end
+
+    private_class_method :get_base_value
+  end
+end

data/lib/lz_string/utf16.rb

@@ -0,0 +1,24 @@
+module LZString
+  # UTF16 compressing algorithm.
+  class UTF16
+    # @param input [String]
+    def self.compress(input)
+      return "" if (input == nil)
+
+      LZString::Base.compress(input, 15, lambda { |a| (a + 32).chr("UTF-8") }) + " "
+    end
+
+    # @param compressed [String]
+    def self.decompress(compressed)
+      return "" if (compressed == nil)
+      return nil if (compressed == "")
+
+      LZString::Base.decompress(
+        compressed.length,
+        16384,
+        lambda { |index| compressed[index].ord - 32 },
+        "UTF-8"
+      )
+    end
+  end
+end

data/lib/lz_string/version.rb

@@ -0,0 +1,4 @@
+module LZString
+  # :nodoc:
+  VERSION = "0.1.2"
+end

data/lib/lz_string.rb

@@ -0,0 +1,23 @@
+require "lz_string/base"
+require "lz_string/base64"
+require "lz_string/utf16"
+require "lz_string/urisafe"
+require "lz_string/version"
+
+# LZ-based compression algorithm.
+module LZString
+  # @param input [String]
+  def self.compress(input)
+    return "" if input.nil?
+
+    LZString::Base.compress(input, 16, lambda { |a| a.chr("UTF-8") })
+  end
+
+  # @param compressed [String]
+  def self.decompress(compressed)
+    return "" if compressed.nil?
+    return nil if compressed == ""
+
+    LZString::Base.decompress(compressed.length, 32768, lambda { |index| compressed[index].ord })
+  end
+end

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification
+name: scrapy_rocket_lz_string
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Altivi
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-12-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-rcov-text
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Ruby implementation of LZ-String compression algorithm
+email:
+- thepoddubstep@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/lz_string.rb
+- lib/lz_string/base.rb
+- lib/lz_string/base64.rb
+- lib/lz_string/urisafe.rb
+- lib/lz_string/utf16.rb
+- lib/lz_string/version.rb
+homepage: https://github.com/Altivi/lz-string
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key:
+specification_version: 4
+summary: Ruby implementation of LZ-String compression algorithm
+test_files: []