ziperu 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -82,6 +82,12 @@ module Ziperu
82
82
  @closed = false
83
83
  @current_entry = nil
84
84
  @written = []
85
+ # On a stream, a file entry is begun in the native writer only when
86
+ # its first bytes are written (or the next entry, or the end, comes):
87
+ # until then nothing of it has gone out, so `rollback_entry` can
88
+ # still take it back. Its name and options wait here.
89
+ @deferred = nil
90
+ @stream_names = {} if @streamed
85
91
  end
86
92
 
87
93
  class << self
@@ -224,6 +230,42 @@ module Ziperu
224
230
  @current_entry = new_entry
225
231
  end
226
232
 
233
+ # Takes the entry being written back out of the archive, as if it had
234
+ # never been put: the central directory will not list it, and the
235
+ # next entry goes where it was. For an entry whose data could not be
236
+ # had (a download that failed, say), so the rest of the archive can
237
+ # still be written:
238
+ #
239
+ # files.each do |name, url|
240
+ # zos.put_next_entry(name)
241
+ # begin
242
+ # fetch(url) { |chunk| zos << chunk }
243
+ # rescue Net::HTTPError
244
+ # zos.rollback_entry
245
+ # end
246
+ # end
247
+ #
248
+ # On a stream to an IO that cannot seek (`body`, `write_buffer` to a
249
+ # pipe) bytes already sent cannot be taken back, so there it works
250
+ # until the entry's first bytes are written, and raises after.
251
+ # Returns the entry taken back.
252
+ # Not in rubyzip.
253
+ def rollback_entry
254
+ check_open!
255
+ entry = @current_entry
256
+ raise Error, 'there is no entry being written to take back' if entry.nil? || !@written.last.equal?(entry)
257
+
258
+ if @deferred
259
+ @deferred = nil
260
+ else
261
+ @writer.rollback_entry
262
+ end
263
+ @written.pop
264
+ @stream_names&.delete(entry.name.b)
265
+ @current_entry = nil
266
+ entry
267
+ end
268
+
227
269
  # Bytes handed to the IO so far, for a stream to an IO that cannot
228
270
  # seek (`write_buffer` to a pipe or socket, `body`): what the consumer
229
271
  # has been given, for a progress report alongside
@@ -320,6 +362,8 @@ module Ziperu
320
362
  { dos_date: time.to_binary_dos_date, dos_time: time.to_binary_dos_time,
321
363
  unix_perms: entry.unix_perms }
322
364
  end
365
+ start_deferred
366
+ @stream_names[(new_name || entry.name).to_s.b] = true if @stream_names
323
367
  @writer.raw_copy(entry.archive, entry.index, new_name, touch_opts)
324
368
  @written << entry
325
369
  @current_entry = nil
@@ -347,6 +391,7 @@ module Ziperu
347
391
  raise Error, "cannot write data to directory entry '#{@current_entry.name}'"
348
392
  end
349
393
 
394
+ start_deferred
350
395
  @writer.write(data.to_s.b)
351
396
  self
352
397
  end
@@ -362,6 +407,7 @@ module Ziperu
362
407
 
363
408
  @writer.comment = comment
364
409
  end
410
+ start_deferred
365
411
  @current_entry = nil
366
412
  begin
367
413
  bytes, stats = @writer.finish
@@ -372,6 +418,10 @@ module Ziperu
372
418
  "the IO given does not allow (#{e.message}); open it for reading and writing"
373
419
  end
374
420
  unless @aborted
421
+ if stats.size != @written.size
422
+ raise Error, "the archive holds #{stats.size} entries where #{@written.size} were written"
423
+ end
424
+
375
425
  stats.each_with_index do |(offset, size, compressed_size, crc), i|
376
426
  @written[i].written_as(offset, size, compressed_size, crc)
377
427
  end
@@ -388,19 +438,40 @@ module Ziperu
388
438
  raise ::Ziperu::CompressionMethodError, entry.compression_method
389
439
  end
390
440
 
441
+ start_deferred
391
442
  opts = native_entry_options(entry)
443
+ if @stream_names
444
+ # The native writer would catch a second entry under a name only
445
+ # once it begins it, which on a stream is later (see @deferred).
446
+ raise EntryExistsError.new('put_next_entry', entry.name) if @stream_names.key?(entry.name.b)
447
+
448
+ @stream_names[entry.name.b] = true
449
+ end
392
450
  if entry.directory?
393
451
  @writer.add_directory(entry.name, opts)
394
452
  elsif entry.symlink?
395
453
  @writer.add_symlink(entry.name, entry.symlink_target, opts)
396
454
  else
397
455
  opts.merge!(@encrypter.native_options)
398
- @writer.start_entry(entry.name.b, opts)
456
+ if @streamed
457
+ @deferred = [entry.name.b, opts]
458
+ else
459
+ @writer.start_entry(entry.name.b, opts)
460
+ end
399
461
  end
400
462
  rescue Native::DuplicateEntryError
401
463
  raise EntryExistsError.new('put_next_entry', entry.name)
402
464
  end
403
465
 
466
+ # Begins in the native writer the entry `init_next_entry` deferred.
467
+ def start_deferred
468
+ name, opts = @deferred
469
+ return unless name
470
+
471
+ @deferred = nil
472
+ @writer.start_entry(name, opts)
473
+ end
474
+
404
475
  def check_open!
405
476
  raise Error, 'zip stream is closed' if @closed
406
477
  end
@@ -0,0 +1,278 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'net/http'
4
+ require 'uri'
5
+
6
+ module Ziperu
7
+ # A read-only, seekable IO over an archive somewhere else: an HTTP(S)
8
+ # URL that answers Range requests (S3, GCS, a CDN, most file servers),
9
+ # or anything a block can fetch byte ranges from. Opening an archive
10
+ # through it reads only what is needed: the end of the file (its
11
+ # central directory) in one request, then each entry's bytes as they
12
+ # are read, so listing a 5 GB archive or pulling one file out of it
13
+ # costs a few requests and a few megabytes.
14
+ #
15
+ # io = Ziperu::RemoteIO.new("https://example.com/big.zip")
16
+ # Ziperu::File.open_buffer(io, write: false) do |zf|
17
+ # zf.extract("docs/readme.txt", "readme.txt")
18
+ # end
19
+ #
20
+ # S3 refuses HEAD to some credentials, so the size comes from the first
21
+ # ranged GET (a presigned URL works as is). With the AWS SDK instead:
22
+ #
23
+ # s3 = Aws::S3::Client.new
24
+ # size = s3.head_object(bucket: b, key: k).content_length
25
+ # io = Ziperu::RemoteIO.new(size: size) do |offset, length|
26
+ # s3.get_object(bucket: b, key: k, range: "bytes=#{offset}-#{offset + length - 1}").body.read
27
+ # end
28
+ #
29
+ # Reads go in blocks of `block_size` bytes (adjacent missing blocks in
30
+ # one request); the last `cache_blocks` of them are kept. Not thread
31
+ # safe: one RemoteIO per thread.
32
+ #
33
+ # Not in rubyzip.
34
+ class RemoteIO
35
+ # Raised when the server or the block does not give the bytes asked
36
+ # for.
37
+ class Error < ::Ziperu::Error; end
38
+
39
+ DEFAULT_BLOCK_SIZE = 256 * 1024
40
+ DEFAULT_CACHE_BLOCKS = 64
41
+ # The first request asks for this much of the end: the end of central
42
+ # directory record, and for most archives the whole central directory.
43
+ TAIL_SIZE = 64 * 1024
44
+ MAX_REDIRECTS = 5
45
+
46
+ attr_reader :size, :pos, :requests
47
+ alias tell pos
48
+ alias length size
49
+
50
+ # `url` (a String or URI) to fetch over HTTP(S) with `headers` sent on
51
+ # every request; or, with no url, `size:` and a block that returns
52
+ # `length` bytes from `offset`.
53
+ def initialize(url = nil, headers: {}, size: nil, block_size: DEFAULT_BLOCK_SIZE,
54
+ cache_blocks: DEFAULT_CACHE_BLOCKS, open_timeout: 10, read_timeout: 60, &fetch)
55
+ raise ArgumentError, 'give a url or a block, not both' if url && fetch
56
+ raise ArgumentError, 'give a url, or size: and a block' unless url || (fetch && size)
57
+ raise ArgumentError, 'block_size must be positive' unless block_size.positive?
58
+
59
+ @block_size = block_size
60
+ @cache_blocks = [cache_blocks, 1].max
61
+ @blocks = {}
62
+ @tail_start = nil
63
+ @tail = nil
64
+ @pos = 0
65
+ @requests = 0
66
+ @closed = false
67
+ if url
68
+ @uri = URI(url.to_s)
69
+ @headers = headers
70
+ @open_timeout = open_timeout
71
+ @read_timeout = read_timeout
72
+ @size = discover_size
73
+ else
74
+ @fetch = fetch
75
+ @size = Integer(size)
76
+ end
77
+ end
78
+
79
+ def read(length = nil, outbuf = nil)
80
+ check_open
81
+ if length.nil?
82
+ data = take(@size - @pos)
83
+ else
84
+ raise ArgumentError, "negative length #{length} given" if length.negative?
85
+ return +''.b if length.zero?
86
+ return nil if @pos >= @size
87
+
88
+ data = take([length, @size - @pos].min)
89
+ end
90
+ outbuf ? outbuf.replace(data) : data
91
+ end
92
+
93
+ def seek(offset, whence = ::IO::SEEK_SET)
94
+ check_open
95
+ target =
96
+ case whence
97
+ when ::IO::SEEK_SET, :SET then offset
98
+ when ::IO::SEEK_CUR, :CUR then @pos + offset
99
+ when ::IO::SEEK_END, :END then @size + offset
100
+ else raise ArgumentError, "unknown whence: #{whence.inspect}"
101
+ end
102
+ raise Errno::EINVAL, "seek to #{target}" if target.negative?
103
+
104
+ @pos = target
105
+ 0
106
+ end
107
+
108
+ def pos=(offset)
109
+ seek(offset)
110
+ end
111
+
112
+ def rewind
113
+ seek(0)
114
+ end
115
+
116
+ def eof?
117
+ @pos >= @size
118
+ end
119
+ alias eof eof?
120
+
121
+ def binmode
122
+ self
123
+ end
124
+
125
+ def close
126
+ @closed = true
127
+ @http&.finish if @http&.started?
128
+ @http = nil
129
+ @blocks.clear
130
+ nil
131
+ end
132
+
133
+ def closed?
134
+ @closed
135
+ end
136
+
137
+ private
138
+
139
+ def check_open
140
+ raise IOError, 'closed stream' if @closed
141
+ end
142
+
143
+ # `count` bytes from the position, which moves past them.
144
+ def take(count)
145
+ return +''.b if count <= 0
146
+
147
+ out = +''.b
148
+ at = @pos
149
+ finish = at + count
150
+ while at < finish
151
+ if @tail_start && at >= @tail_start
152
+ piece = @tail.byteslice(at - @tail_start, finish - at)
153
+ else
154
+ index = at / @block_size
155
+ load_blocks(index, ((finish - 1) / @block_size))
156
+ block = @blocks[index]
157
+ offset = at - (index * @block_size)
158
+ piece = block.byteslice(offset, finish - at)
159
+ # Most recently used last, so the first is the one to drop.
160
+ @blocks[index] = @blocks.delete(index)
161
+ end
162
+ raise Error, "no data at offset #{at}" if piece.nil? || piece.empty?
163
+
164
+ out << piece
165
+ at += piece.bytesize
166
+ end
167
+ @pos = finish
168
+ out
169
+ end
170
+
171
+ # Makes sure blocks `first..last` are cached, fetching each run of
172
+ # missing ones in one request.
173
+ def load_blocks(first, last)
174
+ last = [last, first + @cache_blocks - 1].min
175
+ index = first
176
+ while index <= last
177
+ if @blocks.key?(index)
178
+ index += 1
179
+ next
180
+ end
181
+ run_end = index
182
+ run_end += 1 while run_end < last && !@blocks.key?(run_end + 1)
183
+ offset = index * @block_size
184
+ length = [((run_end + 1) * @block_size), @size].min - offset
185
+ data = fetch(offset, length)
186
+ (index..run_end).each_with_index do |i, n|
187
+ @blocks[i] = data.byteslice(n * @block_size, @block_size)
188
+ end
189
+ @blocks.shift while @blocks.size > @cache_blocks
190
+ index = run_end + 1
191
+ end
192
+ end
193
+
194
+ def fetch(offset, length)
195
+ @requests += 1
196
+ data =
197
+ if @fetch
198
+ @fetch.call(offset, length).to_s.b
199
+ else
200
+ http_range("bytes=#{offset}-#{offset + length - 1}", offset)
201
+ end
202
+ unless data.bytesize == length
203
+ raise Error, "asked for #{length} bytes at offset #{offset}, got #{data.bytesize}"
204
+ end
205
+
206
+ data
207
+ end
208
+
209
+ # The size, from the first request: the last TAIL_SIZE bytes (a
210
+ # suffix range), whose Content-Range carries the total. They are kept,
211
+ # being what opening an archive reads first.
212
+ def discover_size
213
+ @requests += 1
214
+ response = request("bytes=-#{TAIL_SIZE}")
215
+ case response
216
+ when Net::HTTPPartialContent
217
+ total = response['content-range'].to_s[%r{/(\d+)\z}, 1]
218
+ raise Error, "#{@uri} answered without a usable Content-Range" unless total
219
+
220
+ body = response.body.to_s.b
221
+ @tail_start = Integer(total) - body.bytesize
222
+ @tail = body
223
+ Integer(total)
224
+ when Net::HTTPRequestedRangeNotSatisfiable
225
+ # An empty file has no last bytes.
226
+ 0
227
+ when Net::HTTPOK
228
+ # The server ignores Range: the whole file came back; keep it.
229
+ body = response.body.to_s.b
230
+ @tail_start = 0
231
+ @tail = body
232
+ body.bytesize
233
+ else
234
+ raise Error, "#{@uri} answered #{response.code} #{response.message}"
235
+ end
236
+ end
237
+
238
+ def http_range(range, offset)
239
+ response = request(range)
240
+ unless response.kind_of?(Net::HTTPPartialContent)
241
+ raise Error, "#{@uri} answered #{response.code} to a Range request at offset #{offset}"
242
+ end
243
+
244
+ response.body.to_s.b
245
+ end
246
+
247
+ # A GET with `Range: range`, following redirects (the URL redirected
248
+ # to is used from then on, as a presigned S3 link redirects once).
249
+ def request(range, redirects = MAX_REDIRECTS)
250
+ http = connection
251
+ get = Net::HTTP::Get.new(@uri)
252
+ @headers.each { |k, v| get[k.to_s] = v.to_s }
253
+ get['range'] = range
254
+ get['accept-encoding'] = 'identity'
255
+ response = http.request(get)
256
+ return response unless response.kind_of?(Net::HTTPRedirection)
257
+ raise Error, "too many redirects from #{@uri}" if redirects.zero?
258
+
259
+ location = URI.join(@uri.to_s, response['location'].to_s)
260
+ if location.host != @uri.host || location.port != @uri.port || location.scheme != @uri.scheme
261
+ @http&.finish if @http&.started?
262
+ @http = nil
263
+ end
264
+ @uri = location
265
+ request(range, redirects - 1)
266
+ end
267
+
268
+ def connection
269
+ return @http if @http&.started?
270
+
271
+ @http = Net::HTTP.new(@uri.host, @uri.port)
272
+ @http.use_ssl = @uri.scheme == 'https'
273
+ @http.open_timeout = @open_timeout
274
+ @http.read_timeout = @read_timeout
275
+ @http.start
276
+ end
277
+ end
278
+ end
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Makes ziperu answer for rubyzip in this process, whatever else is
4
+ # installed:
5
+ #
6
+ # # config/boot.rb, or the first line of a script
7
+ # require "ziperu/replace_rubyzip"
8
+ #
9
+ # From then on `require "zip"`, `require "rubyzip"` and every
10
+ # `require "zip/..."` load ziperu's files, so `Zip` is `Ziperu` and the
11
+ # gems written against rubyzip (roo, caxlsx, rubyXL, ...) run on it.
12
+ #
13
+ # Under Bundler this is what already happens, since ziperu's `lib` comes
14
+ # first on the load path. This file is for everywhere else: a script
15
+ # without Bundler, where RubyGems would activate the rubyzip gem a
16
+ # dependency asks for and put its files ahead of ziperu's on the load
17
+ # path. It must be required before anything loads rubyzip; if rubyzip is
18
+ # already loaded it raises LoadError rather than mix the two.
19
+ #
20
+ # Not in rubyzip.
21
+
22
+ module Ziperu
23
+ # The `require` that sends rubyzip's files to ziperu's. Not in rubyzip.
24
+ module ReplaceRubyzip
25
+ # ziperu's own `lib`, where `zip.rb`, `rubyzip.rb` and `zip/` live.
26
+ LIB = ::File.expand_path('..', __dir__)
27
+
28
+ # The ziperu file that stands in for the rubyzip feature `name`, if
29
+ # there is one.
30
+ def self.substitute(name)
31
+ name = name.to_s
32
+ name = name.delete_suffix('.rb')
33
+ return unless name == 'zip' || name == 'rubyzip' || name.start_with?('zip/')
34
+
35
+ path = ::File.join(LIB, "#{name}.rb")
36
+ path if ::File.file?(path)
37
+ end
38
+
39
+ # Whether rubyzip itself has already been loaded: a `Zip` that is not
40
+ # ziperu's, or one of rubyzip's files among the loaded features.
41
+ def self.rubyzip_loaded?
42
+ return true if defined?(::Zip) && !(defined?(::Ziperu::File) && ::Zip.equal?(::Ziperu))
43
+
44
+ ours = "#{LIB}/"
45
+ $LOADED_FEATURES.any? do |feature|
46
+ !feature.start_with?(ours) && feature.match?(%r{/(zip|rubyzip)\.rb\z|/zip/[^/]+\.rb\z}) &&
47
+ feature.include?('rubyzip')
48
+ end
49
+ end
50
+
51
+ private
52
+
53
+ def require(name)
54
+ path = ReplaceRubyzip.substitute(name)
55
+ path ? super(path) : super
56
+ end
57
+ end
58
+ end
59
+
60
+ if Ziperu::ReplaceRubyzip.rubyzip_loaded?
61
+ raise LoadError, 'rubyzip is already loaded, so ziperu cannot stand in for it: require ' \
62
+ '"ziperu/replace_rubyzip" before anything that loads rubyzip (first thing ' \
63
+ 'in config/boot.rb, or at the top of the script)'
64
+ end
65
+
66
+ Kernel.prepend(Ziperu::ReplaceRubyzip)
67
+ require 'zip'
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Ziperu
4
- VERSION = "0.2.1"
4
+ VERSION = "0.2.2"
5
5
  # The rubyzip API this gem tracks.
6
6
  RUBYZIP_COMPAT_VERSION = "3.7"
7
7
  end
data/lib/ziperu.rb CHANGED
@@ -82,6 +82,10 @@ require_relative "ziperu/size_estimator"
82
82
  # c.unicode_names = true
83
83
  # end
84
84
  module Ziperu
85
+ # Loaded on first use, so net/http is only required by those who read
86
+ # archives over HTTP.
87
+ autoload :RemoteIO, ::File.expand_path('ziperu/remote_io', __dir__)
88
+
85
89
  # Used by the ported Deflater only; kept because rubyzip exposes it.
86
90
  ZLIB_FLUSHING_STRATEGY = defined?(JRUBY_VERSION) ? Zlib::SYNC_FLUSH : Zlib::NO_FLUSH
87
91
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ziperu
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maor Caplan
@@ -53,6 +53,7 @@ files:
53
53
  - ext/ziperu/src/reader.rs
54
54
  - ext/ziperu/src/sink.rs
55
55
  - ext/ziperu/src/source.rs
56
+ - ext/ziperu/src/stream.rs
56
57
  - ext/ziperu/src/stream_stats.rs
57
58
  - ext/ziperu/src/writer.rs
58
59
  - ext/ziperu/src/zipcrypto.rs
@@ -151,6 +152,8 @@ files:
151
152
  - lib/ziperu/pass_thru_compressor.rb
152
153
  - lib/ziperu/pass_thru_decompressor.rb
153
154
  - lib/ziperu/rails_streaming.rb
155
+ - lib/ziperu/remote_io.rb
156
+ - lib/ziperu/replace_rubyzip.rb
154
157
  - lib/ziperu/settings.rb
155
158
  - lib/ziperu/size_estimator.rb
156
159
  - lib/ziperu/streamable_directory.rb