id_pack 1.0.0 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.editorconfig +15 -0
- data/.gitattributes +1 -0
- data/.github/workflows/tests.yml +34 -0
- data/.gitignore +319 -7
- data/.rubocop.yml +56 -0
- data/Gemfile +6 -1
- data/README.adoc +2 -2
- data/app/assets/javascripts/lib/id-packer.js +300 -5
- data/app/assets/javascripts/lib/uuid-packer.js +1 -1
- data/id_pack.gemspec +7 -5
- data/lib/id_pack/id_packer.rb +90 -63
- data/lib/id_pack/lz_string.rb +83 -83
- data/lib/id_pack/uuid_packer.rb +5 -5
- data/lib/id_pack/version.rb +1 -1
- data/{app/assets/javascripts/lib → vendor/assets/javascripts}/lz-string.js +0 -0
- metadata +18 -17
- data/.travis.yml +0 -5
- data/Gemfile.lock +0 -48
data/lib/id_pack/id_packer.rb
CHANGED
@@ -1,41 +1,41 @@
|
|
1
1
|
module IdPack
|
2
2
|
|
3
|
-
# This is a module to encode an integer array into our compressed format.
|
4
|
-
# Basically there are only 2 methods in this module, encode and decode.
|
5
|
-
#
|
6
|
-
# Usage:
|
7
|
-
# encode:
|
8
|
-
# a usual use case of encode is to provide the server with object ids
|
9
|
-
# that have already been fetched and hence we don't need their data to
|
10
|
-
# be returned
|
11
|
-
#
|
12
|
-
# Example:
|
13
|
-
#
|
14
|
-
# IdPack::IdPacker.new.encode([5, 6, 21, 23, 25]) # => "_F~C_P.V"
|
15
|
-
#
|
16
|
-
# decode:
|
17
|
-
# mainly used by the server to convert the compressed string back into
|
18
|
-
# the integer array
|
19
|
-
#
|
20
|
-
# Example:
|
21
|
-
#
|
22
|
-
# IdPack::IdPacker.new.decode("_F~C_P.V") # => [5, 6, 21, 23, 25]
|
3
|
+
# This is a module to encode an integer array into our compressed format.
|
4
|
+
# Basically there are only 2 methods in this module, encode and decode.
|
5
|
+
#
|
6
|
+
# Usage:
|
7
|
+
# encode:
|
8
|
+
# a usual use case of encode is to provide the server with object ids
|
9
|
+
# that have already been fetched and hence we don't need their data to
|
10
|
+
# be returned
|
11
|
+
#
|
12
|
+
# Example:
|
13
|
+
#
|
14
|
+
# IdPack::IdPacker.new.encode([5, 6, 21, 23, 25]) # => "_F~C_P.V"
|
15
|
+
#
|
16
|
+
# decode:
|
17
|
+
# mainly used by the server to convert the compressed string back into
|
18
|
+
# the integer array
|
19
|
+
#
|
20
|
+
# Example:
|
21
|
+
#
|
22
|
+
# IdPack::IdPacker.new.decode("_F~C_P.V") # => [5, 6, 21, 23, 25]
|
23
23
|
|
24
24
|
class IdPacker
|
25
25
|
|
26
26
|
class InvalidEncodedCharException < StandardError; end
|
27
27
|
|
28
|
-
SPACES_PREFIX = '_'
|
29
|
-
BINARY_PREFIX = '.'
|
30
|
-
RANGE_PREFIX = '~'
|
28
|
+
SPACES_PREFIX = '_'.freeze
|
29
|
+
BINARY_PREFIX = '.'.freeze
|
30
|
+
RANGE_PREFIX = '~'.freeze
|
31
31
|
WINDOW_SIZE = 10
|
32
32
|
EXCLUDE_NIL = true
|
33
|
-
ENCODED_NUMBER_CHARS = (('A'..'Z').to_a + ('a'..'z').to_a + ('0'..'9').to_a).join
|
33
|
+
ENCODED_NUMBER_CHARS = "#{(('A'..'Z').to_a + ('a'..'z').to_a + ('0'..'9').to_a).join}-".freeze
|
34
34
|
|
35
35
|
|
36
36
|
# [5, 6, 21, 23, 25]
|
37
37
|
# => "_F~C_P.V"
|
38
|
-
def encode
|
38
|
+
def encode(array, window_size = WINDOW_SIZE, _exclude_nil = EXCLUDE_NIL, output_charset = ENCODED_NUMBER_CHARS)
|
39
39
|
encoded_array = ''
|
40
40
|
|
41
41
|
ranges = convert_numbers_to_ranges array.uniq.sort
|
@@ -48,7 +48,7 @@ module IdPack
|
|
48
48
|
decimal_number = 0
|
49
49
|
encoded_string = ''
|
50
50
|
|
51
|
-
ranges.each_with_index do |range,
|
51
|
+
ranges.each_with_index do |range, _i|
|
52
52
|
spaces = range.begin - prev_end
|
53
53
|
|
54
54
|
if group_with_prev
|
@@ -56,26 +56,34 @@ module IdPack
|
|
56
56
|
ranges_to_group << range
|
57
57
|
binary_number = convert_ranges_to_binary_number ranges_to_group
|
58
58
|
decimal_number = convert_binary_number_to_decimal_number binary_number
|
59
|
-
encoded_string = BINARY_PREFIX + encode_decimal_number(
|
59
|
+
encoded_string = BINARY_PREFIX + encode_decimal_number(
|
60
|
+
decimal_number, output_charset
|
61
|
+
)
|
60
62
|
encoded_array += encoded_string
|
61
63
|
ranges_to_group = []
|
62
64
|
group_with_prev = false
|
63
65
|
elsif range.end - curr_start + 1 >= window_size
|
64
66
|
if ranges_to_group.length == 1
|
65
|
-
encoded_string = RANGE_PREFIX + encode_decimal_number(
|
67
|
+
encoded_string = RANGE_PREFIX + encode_decimal_number(
|
68
|
+
ranges_to_group.first.size, output_charset
|
69
|
+
)
|
66
70
|
encoded_array += encoded_string
|
67
71
|
else
|
68
72
|
binary_number = convert_ranges_to_binary_number ranges_to_group
|
69
73
|
decimal_number = convert_binary_number_to_decimal_number binary_number
|
70
|
-
encoded_string = BINARY_PREFIX + encode_decimal_number(
|
74
|
+
encoded_string = BINARY_PREFIX + encode_decimal_number(
|
75
|
+
decimal_number, output_charset
|
76
|
+
)
|
71
77
|
encoded_array += encoded_string
|
72
78
|
end
|
73
79
|
ranges_to_group = []
|
74
|
-
encoded_string = SPACES_PREFIX + encode_decimal_number(spaces,
|
80
|
+
encoded_string = SPACES_PREFIX + encode_decimal_number(spaces,
|
81
|
+
output_charset)
|
75
82
|
encoded_array += encoded_string
|
76
83
|
|
77
84
|
if range.size >= window_size
|
78
|
-
encoded_string = RANGE_PREFIX + encode_decimal_number(range.size,
|
85
|
+
encoded_string = RANGE_PREFIX + encode_decimal_number(range.size,
|
86
|
+
output_charset)
|
79
87
|
encoded_array += encoded_string
|
80
88
|
group_with_prev = false
|
81
89
|
else
|
@@ -88,12 +96,14 @@ module IdPack
|
|
88
96
|
end
|
89
97
|
else
|
90
98
|
if spaces >= 0
|
91
|
-
encoded_string = SPACES_PREFIX + encode_decimal_number(spaces,
|
99
|
+
encoded_string = SPACES_PREFIX + encode_decimal_number(spaces,
|
100
|
+
output_charset)
|
92
101
|
encoded_array += encoded_string
|
93
102
|
end
|
94
103
|
|
95
104
|
if range.size >= window_size
|
96
|
-
encoded_string = RANGE_PREFIX + encode_decimal_number(range.size,
|
105
|
+
encoded_string = RANGE_PREFIX + encode_decimal_number(range.size,
|
106
|
+
output_charset)
|
97
107
|
encoded_array += encoded_string
|
98
108
|
else
|
99
109
|
ranges_to_group.push range
|
@@ -106,12 +116,15 @@ module IdPack
|
|
106
116
|
end
|
107
117
|
|
108
118
|
if ranges_to_group.length == 1
|
109
|
-
encoded_string = RANGE_PREFIX + encode_decimal_number(
|
119
|
+
encoded_string = RANGE_PREFIX + encode_decimal_number(
|
120
|
+
ranges_to_group.first.size, output_charset
|
121
|
+
)
|
110
122
|
encoded_array += encoded_string
|
111
|
-
elsif ranges_to_group.length
|
123
|
+
elsif ranges_to_group.length.positive?
|
112
124
|
binary_number = convert_ranges_to_binary_number ranges_to_group
|
113
125
|
decimal_number = convert_binary_number_to_decimal_number binary_number
|
114
|
-
encoded_string = BINARY_PREFIX + encode_decimal_number(decimal_number,
|
126
|
+
encoded_string = BINARY_PREFIX + encode_decimal_number(decimal_number,
|
127
|
+
output_charset)
|
115
128
|
encoded_array += encoded_string
|
116
129
|
end
|
117
130
|
|
@@ -130,8 +143,10 @@ module IdPack
|
|
130
143
|
encoded_caches.each_char do |c|
|
131
144
|
if [SPACES_PREFIX, BINARY_PREFIX, RANGE_PREFIX].include?(c)
|
132
145
|
unless curr_encoded_string_prefix == nil
|
133
|
-
ids_to_include, end_id = convert_encoded_number_to_ids(
|
134
|
-
|
146
|
+
ids_to_include, end_id = convert_encoded_number_to_ids(
|
147
|
+
curr_encoded_string_prefix, encoded_number, start_id
|
148
|
+
)
|
149
|
+
ids.concat(ids_to_include)
|
135
150
|
start_id = end_id + (c == SPACES_PREFIX ? 0 : 1)
|
136
151
|
end
|
137
152
|
curr_encoded_string_prefix = c
|
@@ -143,8 +158,10 @@ module IdPack
|
|
143
158
|
end
|
144
159
|
|
145
160
|
unless curr_encoded_string_prefix == nil
|
146
|
-
ids_to_include, end_id = convert_encoded_number_to_ids(
|
147
|
-
|
161
|
+
ids_to_include, end_id = convert_encoded_number_to_ids(
|
162
|
+
curr_encoded_string_prefix, encoded_number, start_id
|
163
|
+
)
|
164
|
+
ids.concat(ids_to_include)
|
148
165
|
start_id = end_id + 1
|
149
166
|
end
|
150
167
|
|
@@ -166,11 +183,11 @@ module IdPack
|
|
166
183
|
# "encoded_0",diff_last_synced_at_0,\
|
167
184
|
# "encoded_1",diff_last_synced_at_1,\
|
168
185
|
# "encoded_2",diff_last_synced_at_2, ...
|
169
|
-
def encode_sync_str
|
186
|
+
def encode_sync_str(id_synced_at)
|
170
187
|
min_synced_at = id_synced_at.values.min
|
171
188
|
encoded_min_synced_at = LZString.compress_to_encoded_uri_component(min_synced_at.to_s)
|
172
189
|
|
173
|
-
grouped_synced_at = id_synced_at.group_by do |
|
190
|
+
grouped_synced_at = id_synced_at.group_by do |_id, synced_at|
|
174
191
|
synced_at
|
175
192
|
end
|
176
193
|
|
@@ -178,12 +195,19 @@ module IdPack
|
|
178
195
|
ids = ids_group.map do |id_group|
|
179
196
|
int_id = id_group[0].to_s.to_i
|
180
197
|
|
181
|
-
int_id && int_id.to_s == id_group[0].to_s
|
182
|
-
int_id
|
198
|
+
if int_id && int_id.to_s == id_group[0].to_s
|
199
|
+
int_id
|
200
|
+
else
|
183
201
|
id_group[0].to_s
|
202
|
+
end
|
184
203
|
end
|
185
204
|
|
186
|
-
joined_ids = ids.first.is_a?(String)
|
205
|
+
joined_ids = if ids.first.is_a?(String)
|
206
|
+
ids.join("").gsub(/-/,
|
207
|
+
"")
|
208
|
+
else
|
209
|
+
ids.join(",")
|
210
|
+
end
|
187
211
|
|
188
212
|
encoded_indices = LZString.compress_to_encoded_uri_component(joined_ids)
|
189
213
|
diff_synced_at = synced_at - min_synced_at
|
@@ -193,14 +217,14 @@ module IdPack
|
|
193
217
|
end.join(",")
|
194
218
|
end
|
195
219
|
|
196
|
-
def decode_sync_str
|
220
|
+
def decode_sync_str(sync_str, base_timestamp = 0)
|
197
221
|
# format of sync_str:
|
198
222
|
# min_last_synced_at,
|
199
223
|
# "encoded_0", diff_last_requested_at_0,
|
200
224
|
# "encoded_1", diff_last_requested_at_1,
|
201
225
|
# "encoded_2", diff_last_requested_at_2, ...
|
202
226
|
|
203
|
-
sync_str = sync_str.encode('UTF-8', 'UTF-8', :
|
227
|
+
sync_str = sync_str.encode('UTF-8', 'UTF-8', invalid: :replace)
|
204
228
|
|
205
229
|
encoded_min_last_synced_at, *encoded_ranges = sync_str.split(',')
|
206
230
|
min_last_synced_at = LZString.decompress_from_encoded_uri_component(encoded_min_last_synced_at).to_i
|
@@ -219,7 +243,8 @@ module IdPack
|
|
219
243
|
primary_keys.map!(&:to_i)
|
220
244
|
else
|
221
245
|
primary_keys = primary_keys_str.scan(/.{32}/).map do |uuid_str|
|
222
|
-
[uuid_str[0,8], uuid_str[8,4], uuid_str[12,4], uuid_str[16,4],
|
246
|
+
[uuid_str[0, 8], uuid_str[8, 4], uuid_str[12, 4], uuid_str[16, 4],
|
247
|
+
uuid_str[20, 16]].join("-")
|
223
248
|
end
|
224
249
|
end
|
225
250
|
|
@@ -232,7 +257,7 @@ module IdPack
|
|
232
257
|
|
233
258
|
synced_at_map
|
234
259
|
end
|
235
|
-
rescue
|
260
|
+
rescue StandardError
|
236
261
|
# invalid sync_str, return empty map
|
237
262
|
{}
|
238
263
|
end
|
@@ -242,8 +267,8 @@ module IdPack
|
|
242
267
|
|
243
268
|
# [1,2,3,6,7,8]
|
244
269
|
# => [1..3, 6..8]
|
245
|
-
def convert_numbers_to_ranges
|
246
|
-
return [] unless numbers.length
|
270
|
+
def convert_numbers_to_ranges(numbers)
|
271
|
+
return [] unless numbers.length.positive?
|
247
272
|
|
248
273
|
ranges = []
|
249
274
|
range = nil
|
@@ -251,11 +276,13 @@ module IdPack
|
|
251
276
|
numbers.each_with_index do |number, i|
|
252
277
|
range = Range.new(
|
253
278
|
(
|
254
|
-
range && number == numbers[i - 1] + 1
|
255
|
-
range.begin
|
279
|
+
if range && number == numbers[i - 1] + 1
|
280
|
+
range.begin
|
281
|
+
else
|
256
282
|
number
|
283
|
+
end
|
257
284
|
),
|
258
|
-
number
|
285
|
+
number,
|
259
286
|
)
|
260
287
|
|
261
288
|
ranges << range unless numbers[i + 1] && numbers[i + 1] == number + 1
|
@@ -266,11 +293,11 @@ module IdPack
|
|
266
293
|
|
267
294
|
# [1..3, 6..8]
|
268
295
|
# => "11100111"
|
269
|
-
def convert_ranges_to_binary_number
|
296
|
+
def convert_ranges_to_binary_number(ranges)
|
270
297
|
binary_number = ''
|
271
298
|
|
272
299
|
ranges.each_with_index do |range, i|
|
273
|
-
binary_number += '0' * (range.begin - ranges[i - 1].end - 1) if i
|
300
|
+
binary_number += '0' * (range.begin - ranges[i - 1].end - 1) if i.positive?
|
274
301
|
binary_number += '1' * (range.end - range.begin + 1)
|
275
302
|
end
|
276
303
|
|
@@ -279,11 +306,11 @@ module IdPack
|
|
279
306
|
|
280
307
|
# "10101"
|
281
308
|
# => 21
|
282
|
-
def convert_binary_number_to_decimal_number
|
309
|
+
def convert_binary_number_to_decimal_number(binary_number)
|
283
310
|
decimal_number = 0
|
284
311
|
|
285
312
|
binary_number.length.times do |i|
|
286
|
-
decimal_number += 2
|
313
|
+
decimal_number += 2**(binary_number.length - i - 1) * binary_number[i].to_i
|
287
314
|
end
|
288
315
|
|
289
316
|
decimal_number
|
@@ -291,19 +318,19 @@ module IdPack
|
|
291
318
|
|
292
319
|
# 5
|
293
320
|
# => F"
|
294
|
-
def encode_decimal_number
|
295
|
-
return nil if !decimal_number.is_a?(Integer) || decimal_number
|
321
|
+
def encode_decimal_number(decimal_number, output_charset = ENCODED_NUMBER_CHARS)
|
322
|
+
return nil if !decimal_number.is_a?(Integer) || decimal_number.negative?
|
296
323
|
|
297
324
|
encoded_number = ""
|
298
325
|
base = output_charset.length
|
299
326
|
quotient = decimal_number
|
300
327
|
remainder = nil
|
301
328
|
|
302
|
-
|
329
|
+
loop do
|
303
330
|
remainder = quotient % base
|
304
331
|
encoded_number = output_charset[remainder] + encoded_number
|
305
332
|
quotient = (quotient - remainder) / base
|
306
|
-
break if quotient
|
333
|
+
break if quotient.zero?
|
307
334
|
end
|
308
335
|
|
309
336
|
encoded_number
|
@@ -387,7 +414,7 @@ module IdPack
|
|
387
414
|
end_id = start_id + decimal_number - 1
|
388
415
|
end
|
389
416
|
|
390
|
-
[
|
417
|
+
[ids, end_id]
|
391
418
|
end
|
392
419
|
|
393
420
|
end
|