zip_tricks 2.6.0 → 2.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/zip_tricks/block_deflate.rb +4 -4
- data/lib/zip_tricks/block_write.rb +4 -4
- data/lib/zip_tricks/manifest.rb +13 -13
- data/lib/zip_tricks/rack_body.rb +4 -4
- data/lib/zip_tricks/remote_io.rb +13 -12
- data/lib/zip_tricks/remote_uncap.rb +12 -12
- data/lib/zip_tricks/stored_size_estimator.rb +3 -3
- data/lib/zip_tricks/stream_crc32.rb +4 -4
- data/lib/zip_tricks/streamer.rb +18 -18
- data/lib/zip_tricks/write_and_tell.rb +5 -5
- data/lib/zip_tricks.rb +1 -1
- data/spec/zip_tricks/block_deflate_spec.rb +16 -16
- data/spec/zip_tricks/block_write_spec.rb +16 -16
- data/spec/zip_tricks/manifest_spec.rb +11 -11
- data/spec/zip_tricks/rack_body_spec.rb +6 -6
- data/spec/zip_tricks/remote_io_spec.rb +22 -14
- data/spec/zip_tricks/remote_uncap_spec.rb +20 -20
- data/spec/zip_tricks/stored_size_estimator_spec.rb +4 -4
- data/spec/zip_tricks/stream_crc32_spec.rb +7 -7
- data/spec/zip_tricks/streamer_spec.rb +47 -47
- data/spec/zip_tricks/write_and_tell_spec.rb +6 -6
- data/zip_tricks.gemspec +3 -3
- metadata +2 -2
@@ -6,44 +6,44 @@ describe ZipTricks::BlockWrite do
|
|
6
6
|
adapter = described_class.new{|s|
|
7
7
|
blobs << s
|
8
8
|
}
|
9
|
-
|
9
|
+
|
10
10
|
adapter << 'hello'
|
11
11
|
adapter << 'world'
|
12
12
|
adapter << '!'
|
13
|
-
|
13
|
+
|
14
14
|
expect(blobs).to eq(['hello', 'world', '!'])
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
it 'supports chained shovel' do
|
18
18
|
blobs = []
|
19
19
|
adapter = described_class.new{|s|
|
20
20
|
blobs << s
|
21
21
|
}
|
22
|
-
|
22
|
+
|
23
23
|
adapter << 'hello' << 'world' << '!'
|
24
|
-
|
24
|
+
|
25
25
|
expect(blobs).to eq(['hello', 'world', '!'])
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
it 'can write in all possible encodings, even if the strings are frozen' do
|
29
29
|
destination = ''.encode(Encoding::BINARY)
|
30
|
-
|
30
|
+
|
31
31
|
accum_string = ''
|
32
32
|
adapter = described_class.new{|s| accum_string << s }
|
33
|
-
|
33
|
+
|
34
34
|
adapter << 'hello'
|
35
35
|
adapter << 'привет'
|
36
36
|
adapter << 'привет'.freeze
|
37
37
|
adapter << '!'
|
38
38
|
adapter << SecureRandom.random_bytes(1024)
|
39
|
-
|
39
|
+
|
40
40
|
expect(accum_string.bytesize).to eq(1054)
|
41
41
|
end
|
42
|
-
|
42
|
+
|
43
43
|
it 'can be closed' do
|
44
44
|
expect(described_class.new{}.close).to be_nil
|
45
45
|
end
|
46
|
-
|
46
|
+
|
47
47
|
it 'forces the written strings to binary encoding' do
|
48
48
|
blobs = []
|
49
49
|
adapter = described_class.new{|s|
|
@@ -55,7 +55,7 @@ describe ZipTricks::BlockWrite do
|
|
55
55
|
expect(blobs).not_to be_empty
|
56
56
|
blobs.each {|s| expect(s.encoding).to eq(Encoding::BINARY) }
|
57
57
|
end
|
58
|
-
|
58
|
+
|
59
59
|
it 'omits strings of zero length' do
|
60
60
|
blobs = []
|
61
61
|
adapter = described_class.new{|s|
|
@@ -66,7 +66,7 @@ describe ZipTricks::BlockWrite do
|
|
66
66
|
adapter << '!'
|
67
67
|
expect(blobs).to eq(['hello', '!'])
|
68
68
|
end
|
69
|
-
|
69
|
+
|
70
70
|
it 'omits nils' do
|
71
71
|
blobs = []
|
72
72
|
adapter = described_class.new{|s|
|
@@ -77,17 +77,17 @@ describe ZipTricks::BlockWrite do
|
|
77
77
|
adapter << '!'
|
78
78
|
expect(blobs).to eq(['hello', '!'])
|
79
79
|
end
|
80
|
-
|
80
|
+
|
81
81
|
it 'raises a TypeError on specific unsupported methods' do
|
82
82
|
adapter = described_class.new {|s| }
|
83
83
|
expect {
|
84
84
|
adapter.seek(123)
|
85
85
|
}.to raise_error(/non\-rewindable/)
|
86
|
-
|
86
|
+
|
87
87
|
expect {
|
88
88
|
adapter.to_s
|
89
89
|
}.to raise_error(/non\-rewindable/)
|
90
|
-
|
90
|
+
|
91
91
|
expect {
|
92
92
|
adapter.pos = 123
|
93
93
|
}.to raise_error(/non\-rewindable/)
|
@@ -6,53 +6,53 @@ describe ZipTricks::Manifest do
|
|
6
6
|
raw_file_1 = SecureRandom.random_bytes(1024 * 20)
|
7
7
|
raw_file_2 = SecureRandom.random_bytes(1024 * 128)
|
8
8
|
raw_file_3 = SecureRandom.random_bytes(1258695)
|
9
|
-
|
9
|
+
|
10
10
|
manifest, bytesize = described_class.build do | builder |
|
11
11
|
r = builder.add_stored_entry(name: "first-file.bin", size_uncompressed: raw_file_1.size)
|
12
12
|
expect(r).to eq(builder), "add_stored_entry should return self"
|
13
|
-
|
13
|
+
|
14
14
|
builder.add_stored_entry(name: "second-file.bin", size_uncompressed: raw_file_2.size)
|
15
|
-
|
15
|
+
|
16
16
|
r = builder.add_compressed_entry(name: "second-file-comp.bin", size_uncompressed: raw_file_2.size,
|
17
17
|
size_compressed: raw_file_3.size, segment_info: 'http://example.com/second-file-deflated-segment.bin')
|
18
18
|
expect(r).to eq(builder), "add_compressed_entry should return self"
|
19
19
|
end
|
20
|
-
|
20
|
+
|
21
21
|
require 'range_utils'
|
22
|
-
|
22
|
+
|
23
23
|
expect(manifest).to be_kind_of(Array)
|
24
24
|
total_size_of_all_parts = manifest.inject(0) do | total_bytes, span |
|
25
25
|
total_bytes + RangeUtils.size_from_range(span.byte_range_in_zip)
|
26
26
|
end
|
27
27
|
expect(total_size_of_all_parts).to eq(1410595)
|
28
28
|
expect(bytesize).to eq(1410595)
|
29
|
-
|
29
|
+
|
30
30
|
expect(manifest.length).to eq(7)
|
31
|
-
|
31
|
+
|
32
32
|
first_header = manifest[0]
|
33
33
|
expect(first_header.part_type).to eq(:entry_header)
|
34
34
|
expect(first_header.byte_range_in_zip).to eq(0..43)
|
35
35
|
expect(first_header.filename).to eq("first-file.bin")
|
36
36
|
expect(first_header.additional_metadata).to be_nil
|
37
|
-
|
37
|
+
|
38
38
|
first_body = manifest[1]
|
39
39
|
expect(first_body.part_type).to eq(:entry_body)
|
40
40
|
expect(first_body.byte_range_in_zip).to eq(44..20523)
|
41
41
|
expect(first_body.filename).to eq("first-file.bin")
|
42
42
|
expect(first_body.additional_metadata).to be_nil
|
43
|
-
|
43
|
+
|
44
44
|
third_header = manifest[4]
|
45
45
|
expect(third_header.part_type).to eq(:entry_header)
|
46
46
|
expect(third_header.byte_range_in_zip).to eq(151641..151690)
|
47
47
|
expect(third_header.filename).to eq("second-file-comp.bin")
|
48
48
|
expect(third_header.additional_metadata).to eq("http://example.com/second-file-deflated-segment.bin")
|
49
|
-
|
49
|
+
|
50
50
|
third_body = manifest[5]
|
51
51
|
expect(third_body.part_type).to eq(:entry_body)
|
52
52
|
expect(third_body.byte_range_in_zip).to eq(151691..1410385)
|
53
53
|
expect(third_body.filename).to eq("second-file-comp.bin")
|
54
54
|
expect(third_body.additional_metadata).to eq("http://example.com/second-file-deflated-segment.bin")
|
55
|
-
|
55
|
+
|
56
56
|
cd = manifest[-1]
|
57
57
|
expect(cd.part_type).to eq(:central_directory)
|
58
58
|
expect(cd.byte_range_in_zip).to eq(1410386..1410594)
|
@@ -3,22 +3,22 @@ require_relative '../spec_helper'
|
|
3
3
|
describe ZipTricks::RackBody do
|
4
4
|
it 'is usable as a Rack response body, supports each() and close()' do
|
5
5
|
output_buf = Tempfile.new('output')
|
6
|
-
|
6
|
+
|
7
7
|
file_body = SecureRandom.random_bytes(1024 * 1024 + 8981)
|
8
|
-
|
8
|
+
|
9
9
|
body = described_class.new do | zip |
|
10
10
|
zip.add_stored_entry("A file", file_body.bytesize, Zlib.crc32(file_body))
|
11
11
|
zip << file_body
|
12
12
|
end
|
13
|
-
|
13
|
+
|
14
14
|
body.each do | some_data |
|
15
15
|
output_buf << some_data
|
16
16
|
end
|
17
17
|
body.close
|
18
|
-
|
18
|
+
|
19
19
|
output_buf.rewind
|
20
20
|
expect(output_buf.size).to eq(1057667)
|
21
|
-
|
21
|
+
|
22
22
|
per_filename = {}
|
23
23
|
Zip::File.open(output_buf.path) do |zip_file|
|
24
24
|
# Handle entries one by one
|
@@ -27,7 +27,7 @@ describe ZipTricks::RackBody do
|
|
27
27
|
per_filename[entry.name] = entry.get_input_stream.read
|
28
28
|
end
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
expect(per_filename).to have_key('A file')
|
32
32
|
expect(per_filename['A file'].bytesize).to eq(file_body.bytesize)
|
33
33
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe ZipTricks::RemoteIO do
|
4
|
-
|
4
|
+
|
5
5
|
context 'working with the fetcher object' do
|
6
6
|
it 'asks the fetcher object to obtain the object size and the actual data when reading' do
|
7
7
|
mock_fetcher = double(request_object_size: 120, request_range: 'abc')
|
@@ -9,17 +9,17 @@ describe ZipTricks::RemoteIO do
|
|
9
9
|
expect(subject.read(3)).to eq('abc')
|
10
10
|
end
|
11
11
|
end
|
12
|
-
|
12
|
+
|
13
13
|
context 'when it internally addresses a remote resource' do
|
14
14
|
it 'requests the size of the resource once via #request_object_size and does neet to read if resource is empty' do
|
15
15
|
subject = described_class.new
|
16
16
|
expect(subject).to receive(:request_object_size).and_return(0)
|
17
17
|
expect(subject.read).to be_nil
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
it 'performs remote reads when repeatedly requesting the same chunk, via #request_range' do
|
21
21
|
subject = described_class.new
|
22
|
-
|
22
|
+
|
23
23
|
expect(subject).to receive(:request_object_size).and_return(120)
|
24
24
|
allow(subject).to receive(:request_range) {|range|
|
25
25
|
expect(range).to eq(5..14)
|
@@ -31,7 +31,7 @@ describe ZipTricks::RemoteIO do
|
|
31
31
|
end
|
32
32
|
end
|
33
33
|
end
|
34
|
-
|
34
|
+
|
35
35
|
describe '#seek' do
|
36
36
|
context 'with an unsupported mode' do
|
37
37
|
it 'raises an error' do
|
@@ -55,7 +55,7 @@ describe ZipTricks::RemoteIO do
|
|
55
55
|
it 'seens to 10 bytes to the end of the IO' do
|
56
56
|
uncap = described_class.new
|
57
57
|
expect(uncap).to receive(:request_object_size).and_return(100)
|
58
|
-
|
58
|
+
|
59
59
|
mode = IO::SEEK_END
|
60
60
|
offset = -10
|
61
61
|
expect(uncap.seek(-10, IO::SEEK_END)).to eq(0)
|
@@ -70,19 +70,19 @@ describe ZipTricks::RemoteIO do
|
|
70
70
|
@buf.binmode
|
71
71
|
5.times { @buf << Random.new.bytes(1024 * 1024 * 3) }
|
72
72
|
@buf.rewind
|
73
|
-
|
73
|
+
|
74
74
|
@subject = described_class.new
|
75
|
-
|
75
|
+
|
76
76
|
allow(@subject).to receive(:request_object_size).and_return(@buf.size)
|
77
77
|
allow(@subject).to receive(:request_range) {|range|
|
78
78
|
@buf.read[range].tap { @buf.rewind }
|
79
79
|
}
|
80
80
|
end
|
81
|
-
|
81
|
+
|
82
82
|
after :each do
|
83
83
|
@buf.close; @buf.unlink
|
84
84
|
end
|
85
|
-
|
85
|
+
|
86
86
|
context 'without arguments' do
|
87
87
|
it 'reads the entire buffer and alters the position pointer' do
|
88
88
|
expect(@subject.pos).to eq(0)
|
@@ -93,6 +93,14 @@ describe ZipTricks::RemoteIO do
|
|
93
93
|
end
|
94
94
|
|
95
95
|
context 'with length' do
|
96
|
+
it 'supports an unlimited number of reads of size 0 and does not perform remote fetches for them' do
|
97
|
+
expect(@subject).not_to receive(:request_range)
|
98
|
+
20.times do
|
99
|
+
data = @subject.read(0)
|
100
|
+
expect(data).to eq('')
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
96
104
|
it 'returns exact amount of bytes at the start of the buffer' do
|
97
105
|
bytes_read = @subject.read(10)
|
98
106
|
expect(@subject.pos).to eq(10)
|
@@ -102,10 +110,10 @@ describe ZipTricks::RemoteIO do
|
|
102
110
|
|
103
111
|
it 'returns exact amount of bytes from the middle of the buffer' do
|
104
112
|
@subject.seek(456, IO::SEEK_SET)
|
105
|
-
|
113
|
+
|
106
114
|
bytes_read = @subject.read(10)
|
107
115
|
expect(@subject.pos).to eq(456+10)
|
108
|
-
|
116
|
+
|
109
117
|
@buf.seek(456)
|
110
118
|
expect(bytes_read).to eq(@buf.read(10))
|
111
119
|
end
|
@@ -113,11 +121,11 @@ describe ZipTricks::RemoteIO do
|
|
113
121
|
it 'returns the last N bytes it can read' do
|
114
122
|
at_end = @buf.size - 4
|
115
123
|
@subject.seek(at_end, IO::SEEK_SET)
|
116
|
-
|
124
|
+
|
117
125
|
expect(@subject.pos).to eq(15728636)
|
118
126
|
bytes_read = @subject.read(10)
|
119
127
|
expect(@subject.pos).to eq(@buf.size) # Should have moved the pos pointer to the end
|
120
|
-
|
128
|
+
|
121
129
|
expect(bytes_read.bytesize).to eq(4)
|
122
130
|
|
123
131
|
expect(@subject.pos).to eq(@buf.size)
|
@@ -6,16 +6,16 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
6
6
|
after :each do
|
7
7
|
File.unlink('temp.zip') rescue Errno::ENOENT
|
8
8
|
end
|
9
|
-
|
9
|
+
|
10
10
|
it 'returns an array of remote entries that can be used to fetch the segments from within the ZIP' do
|
11
11
|
payload1 = Tempfile.new 'payload1'
|
12
12
|
payload1 << Random.new.bytes((1024 * 1024 * 5) + 10)
|
13
13
|
payload1.flush; payload1.rewind;
|
14
|
-
|
14
|
+
|
15
15
|
payload2 = Tempfile.new 'payload2'
|
16
16
|
payload2 << Random.new.bytes(1024 * 1024 * 3)
|
17
17
|
payload2.flush; payload2.rewind
|
18
|
-
|
18
|
+
|
19
19
|
payload1_crc = Zlib.crc32(payload1.read).tap { payload1.rewind }
|
20
20
|
payload2_crc = Zlib.crc32(payload2.read).tap { payload2.rewind }
|
21
21
|
|
@@ -32,9 +32,9 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
32
32
|
end
|
33
33
|
end
|
34
34
|
payload1.rewind; payload2.rewind
|
35
|
-
|
35
|
+
|
36
36
|
expect(File).to be_exist('temp.zip')
|
37
|
-
|
37
|
+
|
38
38
|
allow_any_instance_of(described_class).to receive(:request_object_size) {
|
39
39
|
File.size('temp.zip')
|
40
40
|
}
|
@@ -44,22 +44,22 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
44
44
|
f.read(range.end - range.begin + 1)
|
45
45
|
end
|
46
46
|
}
|
47
|
-
|
47
|
+
|
48
48
|
payload1.rewind; payload2.rewind
|
49
|
-
|
49
|
+
|
50
50
|
files = described_class.files_within_zip_at('http://fake.example.com')
|
51
51
|
expect(files).to be_kind_of(Array)
|
52
52
|
expect(files.length).to eq(2)
|
53
|
-
|
53
|
+
|
54
54
|
first, second = *files
|
55
|
-
|
55
|
+
|
56
56
|
expect(first.name).to eq('first-file.bin')
|
57
57
|
expect(first.size_uncompressed).to eq(payload1.size)
|
58
58
|
File.open('temp.zip', 'rb') do |readback|
|
59
59
|
readback.seek(first.starts_at_offset, IO::SEEK_SET)
|
60
60
|
expect(readback.read(12)).to eq(payload1.read(12))
|
61
61
|
end
|
62
|
-
|
62
|
+
|
63
63
|
expect(second.name).to eq('second-file.bin')
|
64
64
|
expect(second.size_uncompressed).to eq(payload2.size)
|
65
65
|
File.open('temp.zip', 'rb') do |readback|
|
@@ -67,15 +67,15 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
67
67
|
expect(readback.read(12)).to eq(payload2.read(12))
|
68
68
|
end
|
69
69
|
end
|
70
|
-
|
70
|
+
|
71
71
|
it 'can cope with an empty file within the zip' do
|
72
72
|
payload1 = Tempfile.new 'payload1'
|
73
73
|
payload1.flush; payload1.rewind;
|
74
|
-
|
74
|
+
|
75
75
|
payload2 = Tempfile.new 'payload2'
|
76
76
|
payload2 << Random.new.bytes(1024)
|
77
77
|
payload2.flush; payload2.rewind
|
78
|
-
|
78
|
+
|
79
79
|
payload1_crc = Zlib.crc32(payload1.read).tap { payload1.rewind }
|
80
80
|
payload2_crc = Zlib.crc32(payload2.read).tap { payload2.rewind }
|
81
81
|
|
@@ -90,9 +90,9 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
90
90
|
end
|
91
91
|
end
|
92
92
|
payload1.rewind; payload2.rewind
|
93
|
-
|
93
|
+
|
94
94
|
expect(File).to be_exist('temp.zip')
|
95
|
-
|
95
|
+
|
96
96
|
allow_any_instance_of(described_class).to receive(:request_object_size) {
|
97
97
|
File.size('temp.zip')
|
98
98
|
}
|
@@ -102,22 +102,22 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
102
102
|
f.read(range.end - range.begin + 1)
|
103
103
|
end
|
104
104
|
}
|
105
|
-
|
105
|
+
|
106
106
|
payload1.rewind; payload2.rewind
|
107
|
-
|
107
|
+
|
108
108
|
files = described_class.files_within_zip_at('http://fake.example.com')
|
109
109
|
expect(files).to be_kind_of(Array)
|
110
110
|
expect(files.length).to eq(2)
|
111
|
-
|
111
|
+
|
112
112
|
first, second = *files
|
113
|
-
|
113
|
+
|
114
114
|
expect(first.name).to eq('first-file.bin')
|
115
115
|
expect(first.size_uncompressed).to eq(payload1.size)
|
116
116
|
File.open('temp.zip', 'rb') do |readback|
|
117
117
|
readback.seek(first.starts_at_offset, IO::SEEK_SET)
|
118
118
|
expect(readback.read(0)).to eq(payload1.read(0))
|
119
119
|
end
|
120
|
-
|
120
|
+
|
121
121
|
expect(second.name).to eq('second-file.bin')
|
122
122
|
expect(second.size_uncompressed).to eq(payload2.size)
|
123
123
|
File.open('temp.zip', 'rb') do |readback|
|
@@ -6,17 +6,17 @@ describe ZipTricks::StoredSizeEstimator do
|
|
6
6
|
raw_file_1 = SecureRandom.random_bytes(1024 * 20)
|
7
7
|
raw_file_2 = SecureRandom.random_bytes(1024 * 128)
|
8
8
|
raw_file_3 = SecureRandom.random_bytes(1258695)
|
9
|
-
|
9
|
+
|
10
10
|
predicted_size = described_class.perform_fake_archiving do | estimator |
|
11
11
|
r = estimator.add_stored_entry("first-file.bin", raw_file_1.size)
|
12
12
|
expect(r).to eq(estimator), "add_stored_entry should return self"
|
13
|
-
|
13
|
+
|
14
14
|
estimator.add_stored_entry("second-file.bin", raw_file_2.size)
|
15
|
-
|
15
|
+
|
16
16
|
r = estimator.add_compressed_entry("second-file.bin", raw_file_2.size, raw_file_3.size)
|
17
17
|
expect(r).to eq(estimator), "add_compressed_entry should return self"
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
expect(predicted_size).to eq(1410524)
|
21
21
|
end
|
22
22
|
end
|
@@ -7,29 +7,29 @@ describe ZipTricks::StreamCRC32 do
|
|
7
7
|
via_from_io = described_class.from_io(raw)
|
8
8
|
expect(via_from_io).to eq(crc)
|
9
9
|
end
|
10
|
-
|
10
|
+
|
11
11
|
it 'allows in-place updates' do
|
12
12
|
raw = StringIO.new(SecureRandom.random_bytes(45 * 1024 * 1024))
|
13
13
|
crc = Zlib.crc32(raw.string)
|
14
|
-
|
14
|
+
|
15
15
|
stream_crc = described_class.new
|
16
16
|
stream_crc << raw.read(1024 * 64) until raw.eof?
|
17
17
|
expect(stream_crc.to_i).to eq(crc)
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
it 'supports chained shovel' do
|
21
21
|
str = 'abcdef'
|
22
22
|
crc = Zlib.crc32(str)
|
23
|
-
|
23
|
+
|
24
24
|
stream_crc = described_class.new
|
25
25
|
stream_crc << 'a' << 'b' << 'c' << 'd' << 'e' << 'f'
|
26
|
-
|
26
|
+
|
27
27
|
expect(stream_crc.to_i).to eq(crc)
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
it 'allows in-place update with a known value' do
|
31
31
|
crc = Zlib.crc32
|
32
|
-
|
32
|
+
|
33
33
|
stream_crc = described_class.new
|
34
34
|
stream_crc << "This is some data"
|
35
35
|
stream_crc.append(45678, 12910)
|