mogilefs-client 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +3 -0
- data/.gitignore +1 -0
- data/GIT-VERSION-GEN +1 -1
- data/GNUmakefile +2 -10
- data/History +3 -0
- data/Rakefile +8 -0
- data/bin/mog +18 -38
- data/examples/stale_fid_checker.rb +108 -0
- data/lib/mogilefs.rb +3 -1
- data/lib/mogilefs/admin.rb +2 -1
- data/lib/mogilefs/backend.rb +16 -44
- data/lib/mogilefs/chunker.rb +11 -5
- data/lib/mogilefs/client.rb +6 -1
- data/lib/mogilefs/http_file.rb +51 -88
- data/lib/mogilefs/mogilefs.rb +91 -39
- data/lib/mogilefs/new_file.rb +78 -0
- data/lib/mogilefs/new_file/common.rb +89 -0
- data/lib/mogilefs/new_file/content_range.rb +105 -0
- data/lib/mogilefs/new_file/stream.rb +91 -0
- data/lib/mogilefs/new_file/tempfile.rb +24 -0
- data/lib/mogilefs/new_file/writer.rb +57 -0
- data/lib/mogilefs/pool.rb +18 -10
- data/lib/mogilefs/socket/kgio.rb +4 -3
- data/lib/mogilefs/socket/pure_ruby.rb +3 -3
- data/lib/mogilefs/socket_common.rb +7 -6
- data/test/fresh.rb +5 -2
- data/test/test_backend.rb +16 -8
- data/test/test_mogilefs_integration.rb +121 -0
- data/test/test_mogstored_rack.rb +112 -6
- data/test/test_pool.rb +26 -0
- metadata +13 -10
- data/Manifest.txt +0 -60
- data/examples/mogstored_rack.rb +0 -188
- data/test/test_unit_mogstored_rack.rb +0 -72
data/lib/mogilefs/client.rb
CHANGED
@@ -11,7 +11,9 @@ class MogileFS::Client
|
|
11
11
|
|
12
12
|
attr_reader :backend
|
13
13
|
|
14
|
+
# :stopdoc:
|
14
15
|
attr_accessor :hosts if defined? $TESTING
|
16
|
+
# :startdoc
|
15
17
|
|
16
18
|
##
|
17
19
|
# Creates a new Client. See MogileFS::Backend#initialize for how to specify
|
@@ -24,6 +26,7 @@ class MogileFS::Client
|
|
24
26
|
@hosts = args[:hosts]
|
25
27
|
@readonly = args[:readonly] ? true : false
|
26
28
|
@timeout = args[:timeout]
|
29
|
+
@fail_timeout = args[:fail_timeout]
|
27
30
|
|
28
31
|
reload
|
29
32
|
end
|
@@ -32,7 +35,9 @@ class MogileFS::Client
|
|
32
35
|
# Creates a new MogileFS::Backend.
|
33
36
|
|
34
37
|
def reload
|
35
|
-
@backend = MogileFS::Backend.new
|
38
|
+
@backend = MogileFS::Backend.new(:hosts => @hosts,
|
39
|
+
:timeout => @timeout,
|
40
|
+
:fail_timeout => @fail_timeout)
|
36
41
|
end
|
37
42
|
|
38
43
|
##
|
data/lib/mogilefs/http_file.rb
CHANGED
@@ -1,8 +1,7 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
2
|
# here are internal implementation details, do not use them in your code
|
3
3
|
require 'stringio'
|
4
|
-
require '
|
5
|
-
require 'mogilefs/chunker'
|
4
|
+
require 'mogilefs/new_file'
|
6
5
|
|
7
6
|
##
|
8
7
|
# HTTPFile wraps up the new file operations for storing files onto an HTTP
|
@@ -12,49 +11,7 @@ require 'mogilefs/chunker'
|
|
12
11
|
# create a new file using MogileFS::MogileFS.new_file.
|
13
12
|
#
|
14
13
|
class MogileFS::HTTPFile < StringIO
|
15
|
-
|
16
|
-
class BadResponseError < MogileFS::Error; end
|
17
|
-
class UnparseableResponseError < MogileFS::Error; end
|
18
|
-
class NoStorageNodesError < MogileFS::Error
|
19
|
-
def message; 'Unable to open socket to storage node'; end
|
20
|
-
end
|
21
|
-
class NonRetryableError < MogileFS::Error; end
|
22
|
-
|
23
|
-
class HTTPSock < MogileFS::Socket
|
24
|
-
attr_accessor :start
|
25
|
-
|
26
|
-
# Increase timeout as we become more invested in uploading with
|
27
|
-
# this socket. The server could be experiencing I/O delays
|
28
|
-
# from large uploads because the sysadmin forgot to tune the
|
29
|
-
# VM sysctls for handling large files.
|
30
|
-
def write(buf)
|
31
|
-
timed_write(buf, Time.now - @start + 5.0)
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
# :stopdoc:
|
36
|
-
MD5_TRAILER_NODES = {} # :nodoc: # EXPERIMENTAL
|
37
|
-
class << self
|
38
|
-
attr_accessor :response_timeout_cb
|
39
|
-
end
|
40
|
-
|
41
|
-
# temporary directories (nginx) may not be configured on the
|
42
|
-
# same device, necessitating a time-consuming full file copy
|
43
|
-
# instead of a quick rename(2)/link(2) operation
|
44
|
-
@response_timeout_cb = lambda do |elapsed_time, bytes_uploaded|
|
45
|
-
mbytes_uploaded = bytes_uploaded / (1024.0 * 1024.0)
|
46
|
-
# assumes worst case is 10M/s on the remote storage disk
|
47
|
-
t = mbytes_uploaded * 10 + elapsed_time
|
48
|
-
t < 5 ? 5 : t
|
49
|
-
end
|
50
|
-
# :startdoc:
|
51
|
-
|
52
|
-
##
|
53
|
-
# The URI this file will be stored to.
|
54
|
-
|
55
|
-
attr_reader :uri
|
56
|
-
|
57
|
-
attr_reader :devid
|
14
|
+
include MogileFS::NewFile::Common
|
58
15
|
|
59
16
|
##
|
60
17
|
# The big_io name in case we have file > 256M
|
@@ -67,34 +24,50 @@ class MogileFS::HTTPFile < StringIO
|
|
67
24
|
# Creates a new HTTPFile with MogileFS-specific data. Use
|
68
25
|
# MogileFS::MogileFS#new_file instead of this method.
|
69
26
|
|
70
|
-
def initialize(dests,
|
27
|
+
def initialize(dests, opts = nil)
|
71
28
|
super ""
|
72
|
-
@
|
29
|
+
@md5 = @streaming_io = @big_io = @active = nil
|
73
30
|
@dests = dests
|
31
|
+
@opts = Integer === opts ? { :content_length => opts } : opts
|
74
32
|
end
|
75
33
|
|
76
34
|
def request_put(sock, uri, file_size, input = nil)
|
77
35
|
host_with_port = "#{uri.host}:#{uri.port}"
|
78
|
-
|
79
|
-
|
36
|
+
clen = @opts[:content_length]
|
37
|
+
file_size ||= clen
|
38
|
+
|
39
|
+
content_md5 = @opts[:content_md5]
|
40
|
+
if String === content_md5
|
41
|
+
file_size or
|
42
|
+
raise ArgumentError,
|
43
|
+
":content_length must be specified with :content_md5 String"
|
44
|
+
file_size = "#{file_size}\r\nContent-MD5: #{content_md5}"
|
45
|
+
elsif content_md5.respond_to?(:call) ||
|
46
|
+
:trailer == content_md5 ||
|
47
|
+
MD5_TRAILER_NODES[host_with_port]
|
80
48
|
file_size = nil
|
81
|
-
md5 =
|
49
|
+
@md5 = Digest::MD5.new
|
82
50
|
end
|
83
51
|
|
84
52
|
if file_size
|
85
53
|
sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \
|
86
54
|
"Content-Length: #{file_size}\r\n\r\n")
|
87
|
-
input ? MogileFS.io.copy_stream(@active = input, sock) : yield(sock)
|
55
|
+
rv = input ? MogileFS.io.copy_stream(@active = input, sock) : yield(sock)
|
88
56
|
else
|
89
|
-
trailers = md5 ? "Trailer: Content-MD5\r\n" : ""
|
57
|
+
trailers = @md5 ? "Trailer: Content-MD5\r\n" : ""
|
90
58
|
sock.write("PUT #{uri.request_uri} HTTP/1.1\r\n" \
|
91
59
|
"Host: #{host_with_port}\r\n#{trailers}" \
|
92
60
|
"Transfer-Encoding: chunked\r\n\r\n")
|
93
|
-
tmp = MogileFS::Chunker.new(sock, md5)
|
61
|
+
tmp = MogileFS::Chunker.new(sock, @md5, content_md5)
|
94
62
|
rv = input ? MogileFS.io.copy_stream(@active = input, tmp) : yield(tmp)
|
95
63
|
tmp.flush
|
96
|
-
rv
|
97
64
|
end
|
65
|
+
|
66
|
+
if clen && clen != rv
|
67
|
+
raise MogileFS::SizeMismatchError,
|
68
|
+
":content_length expected: #{clen.inspect}, actual: #{rv.inspect}"
|
69
|
+
end
|
70
|
+
rv
|
98
71
|
end
|
99
72
|
|
100
73
|
def put_streaming_io(sock, uri) # unlikely to be used
|
@@ -121,51 +94,39 @@ class MogileFS::HTTPFile < StringIO
|
|
121
94
|
# Writes an HTTP PUT request to +sock+ to upload the file and
|
122
95
|
# returns file size if the socket finished writing
|
123
96
|
def upload(devid, uri) # :nodoc:
|
124
|
-
|
125
|
-
sock
|
126
|
-
sock.start = start
|
97
|
+
sock = MogileFS::Socket.tcp(uri.host, uri.port)
|
98
|
+
set_socket_options(sock)
|
127
99
|
file_size = length
|
128
100
|
|
129
101
|
if @streaming_io
|
130
102
|
file_size = put_streaming_io(sock, uri)
|
131
103
|
elsif @big_io
|
132
|
-
|
104
|
+
stat = file = size = nil
|
105
|
+
if @big_io.respond_to?(:stat)
|
106
|
+
stat = @big_io.stat
|
107
|
+
elsif String === @big_io || @big_io.respond_to?(:to_path)
|
133
108
|
file = File.open(@big_io)
|
134
109
|
stat = file.stat
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
elsif @big_io.respond_to?(:size)
|
142
|
-
size = @big_io.size
|
143
|
-
end
|
144
|
-
file_size = request_put(sock, uri, size, @big_io)
|
110
|
+
elsif @big_io.respond_to?(:size)
|
111
|
+
size = @big_io.size
|
112
|
+
end
|
113
|
+
if stat && stat.file?
|
114
|
+
size ||= stat.size
|
115
|
+
file ||= @big_io.to_io if @big_io.respond_to?(:to_io)
|
145
116
|
end
|
117
|
+
file_size = request_put(sock, uri, size, file || @big_io)
|
146
118
|
else
|
147
119
|
rewind
|
148
120
|
request_put(sock, uri, file_size, self)
|
149
121
|
end
|
150
122
|
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
when %r{^HTTP/\d\.\d\s+(2\d\d)\s} # success!
|
155
|
-
file_size
|
156
|
-
when nil
|
157
|
-
raise EmptyResponseError, 'Unable to read response line from server'
|
158
|
-
when %r{^HTTP/\d\.\d\s+(\d+)}
|
159
|
-
raise BadResponseError, "HTTP response status from upload: #$1"
|
160
|
-
else
|
161
|
-
raise UnparseableResponseError,
|
162
|
-
"Response line not understood: #{line.inspect}"
|
163
|
-
end
|
164
|
-
rescue => err
|
123
|
+
read_response(sock) # raises on errors
|
124
|
+
file_size
|
125
|
+
rescue SystemCallError, RetryableError => err
|
165
126
|
rewind_or_raise!(uri, err)
|
166
127
|
raise
|
167
128
|
ensure
|
168
|
-
file.close if file
|
129
|
+
file.close if file && @big_io != file
|
169
130
|
sock.close if sock
|
170
131
|
end
|
171
132
|
|
@@ -175,11 +136,8 @@ class MogileFS::HTTPFile < StringIO
|
|
175
136
|
begin
|
176
137
|
uri = URI.parse(path)
|
177
138
|
bytes_uploaded = upload(devid, uri)
|
178
|
-
|
179
|
-
|
180
|
-
rescue NonRetryableError
|
181
|
-
raise
|
182
|
-
rescue => e
|
139
|
+
return create_close(devid, uri, bytes_uploaded)
|
140
|
+
rescue SystemCallError, RetryableError => e
|
183
141
|
errors ||= []
|
184
142
|
errors << "#{path} - #{e.message} (#{e.class})"
|
185
143
|
end
|
@@ -188,4 +146,9 @@ class MogileFS::HTTPFile < StringIO
|
|
188
146
|
raise NoStorageNodesError,
|
189
147
|
"all paths failed with PUT: #{errors.join(', ')}", []
|
190
148
|
end
|
149
|
+
|
150
|
+
def close
|
151
|
+
commit
|
152
|
+
super
|
153
|
+
end
|
191
154
|
end
|
data/lib/mogilefs/mogilefs.rb
CHANGED
@@ -34,15 +34,41 @@ class MogileFS::MogileFS < MogileFS::Client
|
|
34
34
|
# The domain of keys for this MogileFS client.
|
35
35
|
attr_accessor :domain
|
36
36
|
|
37
|
-
# The timeout for get_file_data
|
37
|
+
# The timeout for get_file_data (per-read() system call).
|
38
|
+
# Defaults to five seconds.
|
38
39
|
attr_accessor :get_file_data_timeout
|
39
40
|
|
41
|
+
# The maximum allowed time for creating a new_file. Defaults to 1 hour.
|
42
|
+
attr_accessor :new_file_max_time
|
43
|
+
|
40
44
|
# Creates a new MogileFS::MogileFS instance. +args+ must include a key
|
41
45
|
# :domain specifying the domain of this client.
|
46
|
+
#
|
47
|
+
# Optional parameters for +args+:
|
48
|
+
#
|
49
|
+
# [:get_file_data_timeout => Numeric]
|
50
|
+
#
|
51
|
+
# See get_file_data_timeout
|
52
|
+
#
|
53
|
+
# [:new_file_max_time => Numeric]
|
54
|
+
#
|
55
|
+
# See new_file_max_time
|
56
|
+
#
|
57
|
+
# [:fail_timeout => Numeric]
|
58
|
+
#
|
59
|
+
# Delay before retrying a failed tracker backends.
|
60
|
+
# Defaults to 5 seconds.
|
61
|
+
#
|
62
|
+
# [:timeout => Numeric]
|
63
|
+
#
|
64
|
+
# Timeout for tracker backend responses.
|
65
|
+
# Defaults to 3 seconds.
|
66
|
+
#
|
42
67
|
def initialize(args = {})
|
43
68
|
@domain = args[:domain]
|
44
69
|
|
45
|
-
@get_file_data_timeout = 5
|
70
|
+
@get_file_data_timeout = args[:get_file_data_timeout] || 5
|
71
|
+
@new_file_max_time = args[:new_file_max_time] || 3600.0
|
46
72
|
|
47
73
|
raise ArgumentError, "you must specify a domain" unless @domain
|
48
74
|
|
@@ -113,11 +139,8 @@ class MogileFS::MogileFS < MogileFS::Client
|
|
113
139
|
|
114
140
|
# Returns +true+ if +key+ exists, +false+ if not
|
115
141
|
def exist?(key)
|
116
|
-
|
117
|
-
|
118
|
-
@backend.pipeline_dispatch(:get_paths, args) { |x| rv = (Hash === x) }
|
119
|
-
@backend.pipeline_wait(1)
|
120
|
-
rv
|
142
|
+
args = { :key => key, :domain => @domain , :ruby_no_raise => true}
|
143
|
+
Hash === @backend.get_paths(args)
|
121
144
|
end
|
122
145
|
|
123
146
|
# Get the URIs for +key+ (paths) as URI::HTTP objects
|
@@ -125,20 +148,45 @@ class MogileFS::MogileFS < MogileFS::Client
|
|
125
148
|
get_paths(key, *args).map! { |path| URI.parse(path) }
|
126
149
|
end
|
127
150
|
|
128
|
-
# Creates a new file +key+ in
|
129
|
-
#
|
130
|
-
#
|
131
|
-
#
|
132
|
-
|
151
|
+
# Creates a new file +key+ in the domain of this object.
|
152
|
+
#
|
153
|
+
# +bytes+ is the expected size of the file if known in advance
|
154
|
+
#
|
155
|
+
# It operates like File.open(..., "w") and may take an optional
|
156
|
+
# block, yielding an IO-like object with support for the methods
|
157
|
+
# documented in MogileFS::NewFile::Writer.
|
158
|
+
#
|
159
|
+
# This atomically replaces existing data stored as +key+
|
160
|
+
# when the block exits or when the returned object is closed.
|
161
|
+
#
|
162
|
+
# +args+ may contain the following options:
|
163
|
+
#
|
164
|
+
# [:content_length => Integer]
|
165
|
+
#
|
166
|
+
# This has the same effect as the (deprecated) +bytes+ parameter.
|
167
|
+
#
|
168
|
+
# [ :largefile => :stream, :content_range or :tempfile ]
|
169
|
+
#
|
170
|
+
# See MogileFS::NewFile for more information on this
|
171
|
+
#
|
172
|
+
# [ :class => String]
|
173
|
+
#
|
174
|
+
# The MogileFS storage class of the object.
|
175
|
+
def new_file(key, args = nil, bytes = nil) # :yields: file
|
133
176
|
raise MogileFS::ReadOnlyError if readonly?
|
134
|
-
opts = { :
|
135
|
-
|
177
|
+
opts = { :key => key, :multi_dest => 1 }
|
178
|
+
case args
|
179
|
+
when Hash
|
180
|
+
opts[:domain] = args[:domain]
|
181
|
+
klass = args[:class] and "default" != klass and opts[:class] = klass
|
182
|
+
when String
|
183
|
+
opts[:class] = args if "default" != args
|
184
|
+
end
|
185
|
+
opts[:domain] ||= @domain
|
136
186
|
res = @backend.create_open(opts)
|
137
187
|
|
138
188
|
dests = if dev_count = res['dev_count'] # multi_dest succeeded
|
139
|
-
(1..dev_count.to_i).map
|
140
|
-
[res["devid_#{i}"], res["path_#{i}"]]
|
141
|
-
end
|
189
|
+
(1..dev_count.to_i).map { |i| [res["devid_#{i}"], res["path_#{i}"]] }
|
142
190
|
else # single destination returned
|
143
191
|
# 0x0040: d0e4 4f4b 2064 6576 6964 3d31 2666 6964 ..OK.devid=1&fid
|
144
192
|
# 0x0050: 3d33 2670 6174 683d 6874 7470 3a2f 2f31 =3&path=http://1
|
@@ -149,19 +197,23 @@ class MogileFS::MogileFS < MogileFS::Client
|
|
149
197
|
[[res['devid'], res['path']]]
|
150
198
|
end
|
151
199
|
|
200
|
+
opts.merge!(args) if Hash === args
|
201
|
+
opts[:backend] = @backend
|
202
|
+
opts[:fid] = res['fid']
|
203
|
+
opts[:content_length] ||= bytes if bytes
|
204
|
+
opts[:new_file_max_time] ||= @new_file_max_time
|
205
|
+
opts[:start_time] = Time.now
|
206
|
+
|
152
207
|
case (dests[0][1] rescue nil)
|
153
|
-
when
|
154
|
-
http_file = MogileFS::
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
:size => rv)
|
163
|
-
rv
|
164
|
-
when nil, '' then
|
208
|
+
when %r{\Ahttp://}
|
209
|
+
http_file = MogileFS::NewFile.new(dests, opts)
|
210
|
+
if block_given?
|
211
|
+
yield http_file
|
212
|
+
return http_file.commit # calls create_close
|
213
|
+
else
|
214
|
+
return http_file
|
215
|
+
end
|
216
|
+
when nil, ''
|
165
217
|
raise MogileFS::EmptyPathError,
|
166
218
|
"Empty path for mogile upload res=#{res.inspect}"
|
167
219
|
else
|
@@ -174,18 +226,20 @@ class MogileFS::MogileFS < MogileFS::Client
|
|
174
226
|
# either a path name (String or Pathname object) or an IO-like object that
|
175
227
|
# responds to #read or #readpartial. Returns size of +file+ stored.
|
176
228
|
# This atomically replaces existing data stored as +key+
|
177
|
-
def store_file(key, klass, file)
|
229
|
+
def store_file(key, klass, file, opts = nil)
|
178
230
|
raise MogileFS::ReadOnlyError if readonly?
|
231
|
+
(opts ||= {})[:class] = klass if String === klass
|
179
232
|
|
180
|
-
new_file(key,
|
233
|
+
new_file(key, opts) { |mfp| mfp.big_io = file }
|
181
234
|
end
|
182
235
|
|
183
236
|
# Stores +content+ into +key+ in class +klass+, where +content+ is a String
|
184
237
|
# This atomically replaces existing data stored as +key+
|
185
|
-
def store_content(key, klass, content)
|
238
|
+
def store_content(key, klass, content, opts = nil)
|
186
239
|
raise MogileFS::ReadOnlyError if readonly?
|
240
|
+
(opts ||= {})[:class] = klass if String === klass
|
187
241
|
|
188
|
-
new_file
|
242
|
+
new_file(key, opts) do |mfp|
|
189
243
|
if content.is_a?(MogileFS::Util::StoreContent)
|
190
244
|
mfp.streaming_io = content
|
191
245
|
else
|
@@ -236,12 +290,10 @@ class MogileFS::MogileFS < MogileFS::Client
|
|
236
290
|
@backend.respond_to?(:_list_keys) and
|
237
291
|
return @backend._list_keys(domain, prefix, after, limit, &block)
|
238
292
|
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
return
|
244
|
-
end
|
293
|
+
res = @backend.list_keys(:domain => domain, :prefix => prefix,
|
294
|
+
:after => after, :limit => limit,
|
295
|
+
:ruby_no_raise => true)
|
296
|
+
MogileFS::Backend::NoneMatchError === res and return
|
245
297
|
|
246
298
|
keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
|
247
299
|
if block
|
@@ -0,0 +1,78 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
#
|
3
|
+
# The MogileFS::MogileFS#new_file method is enhanced in v3.1.0+
|
4
|
+
# to support the :largefile parameter. While we have always
|
5
|
+
# supported large files via the "store_file" method, streaming
|
6
|
+
# large amounts of content of an unknown length required the use
|
7
|
+
# of awkward APIs.
|
8
|
+
#
|
9
|
+
# It is possible to stream large content of known length any WebDAV server.
|
10
|
+
# One example of this is for mirroring a large file from an existing HTTP
|
11
|
+
# server to \MogileFS without letting it hit the local filesystem.
|
12
|
+
#
|
13
|
+
# uri = URI('http://example.com/large_file')
|
14
|
+
# Net::HTTP.start(uri.host, uri.port) do |http|
|
15
|
+
# req = Net::HTTP::Get.new(uri.request_uri)
|
16
|
+
#
|
17
|
+
# http.request(req) do |response|
|
18
|
+
# if len = response.content_length
|
19
|
+
# io = mg.new_file('key', :largefile => true, :content_length => len)
|
20
|
+
# else
|
21
|
+
# warn "trying to upload with Transfer-Encoding: chunked"
|
22
|
+
# warn "this is not supported by all WebDAV servers"
|
23
|
+
# io = mg.new_file('key', :largefile => :stream)
|
24
|
+
# end
|
25
|
+
# response.read_body { |buf| io.write(buf) }
|
26
|
+
# io.close
|
27
|
+
# end
|
28
|
+
# end
|
29
|
+
#
|
30
|
+
# If your WebDAV servers have chunked PUT support (e.g. Perlbal), you can
|
31
|
+
# stream a file of unknown length using "Transfer-Encoding: chunked".
|
32
|
+
#
|
33
|
+
# nf = mg.new_file("key", :largefile => :stream)
|
34
|
+
# nf.write "hello"
|
35
|
+
# nf.write ...
|
36
|
+
# nf.close
|
37
|
+
#
|
38
|
+
# If your WebDAV server has partial PUT support (e.g Apache), you can
|
39
|
+
# you can use multiple PUT requests with "Content-Range" support.
|
40
|
+
# This method is slower than Transfer-Encoding: chunked.
|
41
|
+
#
|
42
|
+
# nf = mg.new_file("key", :largefile => :content_range)
|
43
|
+
# nf.write "hello"
|
44
|
+
# nf.write ...
|
45
|
+
# nf.close
|
46
|
+
#
|
47
|
+
# Finally, if your WebDAV servers does not support either partial nor
|
48
|
+
# nor chunked PUTs, you must buffer a large file of unknown length
|
49
|
+
# using a Tempfile:
|
50
|
+
#
|
51
|
+
# nf = mg.new_file("key", :largefile => :tempfile)
|
52
|
+
# nf.write "hello"
|
53
|
+
# nf.write ...
|
54
|
+
# nf.close
|
55
|
+
#
|
56
|
+
module MogileFS::NewFile
|
57
|
+
|
58
|
+
# avoiding autoload for new code since it's going away in Ruby...
|
59
|
+
def self.new(dests, opts) # :nodoc:
|
60
|
+
largefile = opts[:largefile]
|
61
|
+
largefile = :stream if largefile && opts[:content_length]
|
62
|
+
require "mogilefs/new_file/#{largefile}" if Symbol === largefile
|
63
|
+
case largefile
|
64
|
+
when nil, false
|
65
|
+
MogileFS::HTTPFile
|
66
|
+
when :stream
|
67
|
+
Stream
|
68
|
+
when :content_range
|
69
|
+
ContentRange
|
70
|
+
when :tempfile
|
71
|
+
Tempfile
|
72
|
+
else
|
73
|
+
raise ArgumentError, "largefile: #{largefile.inspect} not understood"
|
74
|
+
end.new(dests, opts)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
require 'mogilefs/new_file/common'
|