id_pack 0.1.0

data/lib/id_pack/id_packer.rb

@@ -0,0 +1,395 @@
+module IdPack
+
+# This is a module to encode an integer array into our compressed format.
+# Basically there are only 2 methods in this module, encode and decode.
+#
+# Usage:
+#   encode:
+#     a usual use case of encode is to provide the server with object ids
+#     that have already been fetched and hence we don't need their data to
+#     be returned
+#
+#     Example:
+#
+#       IdPack::IdPacker.encode([5, 6, 21, 23, 25]) # => "_F~C_P.V"
+#
+#   decode:
+#     mainly used by the server to convert the compressed string back into
+#     the integer array
+#
+#       Example:
+#
+#         IdPack::IdPacker.decode("_F~C_P.V") # => [5, 6, 21, 23, 25]
+
+  class IdPacker
+
+    class InvalidEncodedCharException < StandardError; end
+
+    SPACES_PREFIX = '_'
+    BINARY_PREFIX = '.'
+    RANGE_PREFIX  = '~'
+    WINDOW_SIZE = 10
+    EXCLUDE_NIL = true
+    ENCODED_NUMBER_CHARS = (('A'..'Z').to_a + ('a'..'z').to_a + ('0'..'9').to_a).join + '-'
+
+
+    # [5, 6, 21, 23, 25]
+    # => "_F~C_P.V"
+    def encode array, window_size = WINDOW_SIZE, exclude_nil = EXCLUDE_NIL, output_charset = ENCODED_NUMBER_CHARS
+      encoded_array = ''
+
+      ranges = convert_numbers_to_ranges array.uniq.sort
+      prev_end = 0
+      curr_start = 1
+      spaces = 0
+      group_with_prev = false
+      ranges_to_group = []
+      binary_number = ''
+      decimal_number = 0
+      encoded_string = ''
+
+      ranges.each_with_index do |range, i|
+        spaces = range.begin - prev_end
+
+        if group_with_prev
+          if range.end - curr_start + 1 == window_size
+            ranges_to_group << range
+            binary_number = convert_ranges_to_binary_number ranges_to_group
+            decimal_number = convert_binary_number_to_decimal_number binary_number
+            encoded_string = BINARY_PREFIX + encode_decimal_number(decimal_number, output_charset)
+            encoded_array += encoded_string
+            ranges_to_group = []
+            group_with_prev = false
+          elsif range.end - curr_start + 1 >= window_size
+            if ranges_to_group.length == 1
+              encoded_string = RANGE_PREFIX + encode_decimal_number(ranges_to_group.first.size, output_charset)
+              encoded_array += encoded_string
+            else
+              binary_number = convert_ranges_to_binary_number ranges_to_group
+              decimal_number = convert_binary_number_to_decimal_number binary_number
+              encoded_string = BINARY_PREFIX + encode_decimal_number(decimal_number, output_charset)
+              encoded_array += encoded_string
+            end
+            ranges_to_group = []
+            encoded_string = SPACES_PREFIX + encode_decimal_number(spaces, output_charset)
+            encoded_array += encoded_string
+
+            if range.size >= window_size
+              encoded_string = RANGE_PREFIX + encode_decimal_number(range.size, output_charset)
+              encoded_array += encoded_string
+              group_with_prev = false
+            else
+              ranges_to_group.push range
+              curr_start = range.begin
+              group_with_prev = true
+            end
+          else
+            ranges_to_group.push range
+          end
+        else
+          if spaces >= 0
+            encoded_string = SPACES_PREFIX + encode_decimal_number(spaces, output_charset)
+            encoded_array += encoded_string
+          end
+
+          if range.size >= window_size
+            encoded_string = RANGE_PREFIX + encode_decimal_number(range.size, output_charset)
+            encoded_array += encoded_string
+          else
+            ranges_to_group.push range
+            curr_start = range.begin
+            group_with_prev = true
+          end
+        end
+
+        prev_end = range.end
+      end
+
+      if ranges_to_group.length == 1
+        encoded_string = RANGE_PREFIX + encode_decimal_number(ranges_to_group.first.size, output_charset)
+        encoded_array += encoded_string
+      elsif ranges_to_group.length > 0
+        binary_number = convert_ranges_to_binary_number ranges_to_group
+        decimal_number = convert_binary_number_to_decimal_number binary_number
+        encoded_string = BINARY_PREFIX + encode_decimal_number(decimal_number, output_charset)
+        encoded_array += encoded_string
+      end
+
+      encoded_array
+    end
+
+    # "_F~C_P.V"
+    # => [5, 6, 21, 23, 25]
+    def decode(encoded_caches)
+      curr_encoded_string_prefix = nil
+
+      ids = []
+      start_id = 0
+      encoded_number = ''
+
+      encoded_caches.each_char do |c|
+        if [SPACES_PREFIX, BINARY_PREFIX, RANGE_PREFIX].include?(c)
+          unless curr_encoded_string_prefix == nil
+            ids_to_include, end_id = convert_encoded_number_to_ids(curr_encoded_string_prefix, encoded_number, start_id)
+            ids = ids.concat(ids_to_include)
+            start_id = end_id + (c == SPACES_PREFIX ? 0 : 1)
+          end
+          curr_encoded_string_prefix = c
+          encoded_number = ''
+        else
+          encoded_number = encoded_number + c
+        end
+
+      end
+
+      unless curr_encoded_string_prefix == nil
+        ids_to_include, end_id = convert_encoded_number_to_ids(curr_encoded_string_prefix, encoded_number, start_id)
+        ids = ids.concat(ids_to_include)
+        start_id = end_id + 1
+      end
+
+      ids
+    rescue InvalidEncodedCharException
+      # corrupted encoded_caches, assume nothing cached
+      []
+    end
+
+    # Input: id_synced_at:
+    # {
+    #   1 => synced_at_1_timestamp,
+    #   2 => synced_at_2_timestamp,
+    #   10 => synced_at_10_timestamp, ...
+    # }
+    #
+    # Expected output of sync_str:
+    # min_last_synced_at,\
+    # "encoded_0",diff_last_synced_at_0,\
+    # "encoded_1",diff_last_synced_at_1,\
+    # "encoded_2",diff_last_synced_at_2, ...
+    def encode_sync_str id_synced_at
+      min_synced_at = id_synced_at.values.min
+      encoded_min_synced_at = LZString.compress_to_encoded_uri_component(min_synced_at.to_s)
+
+      grouped_synced_at = id_synced_at.group_by do |id, synced_at|
+        synced_at
+      end
+
+      grouped_synced_at.inject([encoded_min_synced_at]) do |sync_str_arr, (synced_at, ids_group)|
+        ids = ids_group.map do |id_group|
+          int_id = id_group[0].to_s.to_i
+
+          int_id && int_id.to_s == id_group[0].to_s ?
+            int_id :
+            id_group[0].to_s
+        end
+
+        joined_ids = ids.first.is_a?(String) ? ids.join("").gsub(/-/, "") : ids.join(",")
+
+        encoded_indices = LZString.compress_to_encoded_uri_component(joined_ids)
+        diff_synced_at = synced_at - min_synced_at
+        encoded_diff_synced_at = LZString.compress_to_encoded_uri_component(diff_synced_at.to_s)
+
+        sync_str_arr << "#{encoded_indices},#{encoded_diff_synced_at}"
+      end.join(",")
+    end
+
+    def decode_sync_str sync_str, base_timestamp = 0
+      # format of sync_str:
+      # min_last_synced_at,
+      # "encoded_0", diff_last_requested_at_0,
+      # "encoded_1", diff_last_requested_at_1,
+      # "encoded_2", diff_last_requested_at_2, ...
+
+      sync_str = sync_str.encode('UTF-8', 'UTF-8', :invalid => :replace)
+
+      encoded_min_last_synced_at, *encoded_ranges = sync_str.split(',')
+      min_last_synced_at = LZString.decompress_from_encoded_uri_component(encoded_min_last_synced_at).to_i
+
+      grouped_encoded_ranges = encoded_ranges.inject([]) do |grouped, encoded_range|
+        grouped << [] if grouped.last.nil? || grouped.last.length >= 2
+        grouped.last << encoded_range
+        grouped
+      end
+
+      grouped_encoded_ranges.inject({}) do |synced_at_map, (encoded_caches, encoded_diff_last_synced_at)|
+        primary_keys_str = LZString.decompress_from_encoded_uri_component(encoded_caches)
+        primary_keys = primary_keys_str.split(",")
+
+        if primary_keys.first.to_i.to_s == primary_keys.first
+          primary_keys.map!(&:to_i)
+        else
+          primary_keys = primary_keys_str.scan(/.{32}/).map do |uuid_str|
+            [uuid_str[0,8], uuid_str[8,4], uuid_str[12,4], uuid_str[16,4], uuid_str[20,16]].join("-")
+          end
+        end
+
+        diff_last_synced_at = LZString.decompress_from_encoded_uri_component(encoded_diff_last_synced_at).to_i
+        last_synced_at = min_last_synced_at + diff_last_synced_at + base_timestamp
+
+        primary_keys.each do |key|
+          synced_at_map[key] = [synced_at_map[key], last_synced_at].compact.max
+        end
+
+        synced_at_map
+      end
+    rescue
+      # invalid sync_str, return empty map
+      {}
+    end
+
+
+    private
+
+    # [1,2,3,6,7,8]
+    # => [1..3, 6..8]
+    def convert_numbers_to_ranges numbers
+      return [] unless numbers.length > 0
+
+      ranges = []
+      range = nil
+
+      numbers.each_with_index do |number, i|
+        range = Range.new(
+          (
+            range && number == numbers[i - 1] + 1 ?
+              range.begin :
+              number
+          ),
+          number
+        )
+
+        ranges << range unless numbers[i + 1] && numbers[i + 1] == number + 1
+      end
+
+      ranges
+    end
+
+    # [1..3, 6..8]
+    # => "11100111"
+    def convert_ranges_to_binary_number ranges
+      binary_number = ''
+
+      ranges.each_with_index do |range, i|
+        binary_number += '0' * (range.begin - ranges[i - 1].end - 1) if i > 0
+        binary_number += '1' * (range.end - range.begin + 1)
+      end
+
+      binary_number
+    end
+
+    # "10101"
+    # => 21
+    def convert_binary_number_to_decimal_number binary_number
+      decimal_number = 0
+
+      binary_number.length.times do |i|
+        decimal_number += 2 ** (binary_number.length - i - 1) * binary_number[i].to_i
+      end
+
+      decimal_number
+    end
+
+    # 5
+    # => F"
+    def encode_decimal_number decimal_number, output_charset = ENCODED_NUMBER_CHARS
+      return nil if !decimal_number.is_a?(Integer) || decimal_number < 0
+
+      encoded_number = ""
+      base = output_charset.length
+      quotient = decimal_number
+      remainder = nil
+
+      while true do
+        remainder = quotient % base
+        encoded_number = output_charset[remainder] + encoded_number
+        quotient = (quotient - remainder) / base
+        break if quotient == 0
+      end
+
+      encoded_number
+    end
+    alias_method :encode_integer, :encode_decimal_number
+
+    # 21
+    # => "10101"
+    def convert_decimal_number_to_binary_number(decimal_number)
+      binary_number = ""
+      base = 2
+      quotient = decimal_number
+      remainder = 0
+
+      while quotient != 0
+        remainder = quotient % base
+        binary_number = remainder.to_s + binary_number
+        quotient = (quotient - remainder) / base
+      end
+
+      binary_number
+    end
+
+    # "F"
+    # => 5
+    def convert_encoded_number_to_decimal_number(encoded_number)
+      decimal_number = 0
+      index = 0
+
+      encoded_number.each_char do |c|
+        char_index = ENCODED_NUMBER_CHARS.index(c)
+
+        # current char not found in chars, implies corrupted encoded_caches
+        raise InvalidEncodedCharException if char_index.nil?
+
+        decimal_number += ENCODED_NUMBER_CHARS.length**(encoded_number.length - index - 1) * char_index
+        index += 1
+      end
+
+      decimal_number
+    end
+    alias_method :decode_integer, :convert_encoded_number_to_decimal_number
+
+    # encoded_string_prefix, encoded_number, start_id
+    # => [ids_to_include, end_id]
+    #
+    # "_", "E", 1
+    # => [[], 4]
+    #
+    # "~", "C", 5
+    # => [[5, 6], 6]
+    #
+    # "_", "O", 7
+    # => [[], 20]
+    #
+    # ".", "V", 21
+    # => [[21, 23, 25], 25]
+    def convert_encoded_number_to_ids(encoded_string_prefix, encoded_number, start_id)
+      ids = []
+
+      case encoded_string_prefix
+      when SPACES_PREFIX
+        decimal_number = convert_encoded_number_to_decimal_number(encoded_number)
+        end_id = start_id + decimal_number - 1
+      when BINARY_PREFIX
+        decimal_number = convert_encoded_number_to_decimal_number(encoded_number)
+        binary_number = convert_decimal_number_to_binary_number(decimal_number)
+        id = start_id
+        binary_number.each_char do |c|
+          if c == '1'
+            ids << id
+          end
+          id = id + 1
+        end
+        end_id = id - 1
+      when RANGE_PREFIX
+        decimal_number = convert_encoded_number_to_decimal_number(encoded_number)
+        (start_id..(start_id + decimal_number - 1)).each do |id|
+          ids << id
+        end
+        end_id = start_id + decimal_number - 1
+      end
+
+      [ ids, end_id ]
+    end
+
+  end
+
+end