mogilefs-client 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,7 +11,9 @@ class MogileFS::Client
11
11
 
12
12
  attr_reader :backend
13
13
 
14
+ # :stopdoc:
14
15
  attr_accessor :hosts if defined? $TESTING
16
+ # :startdoc
15
17
 
16
18
  ##
17
19
  # Creates a new Client. See MogileFS::Backend#initialize for how to specify
@@ -24,6 +26,7 @@ class MogileFS::Client
24
26
  @hosts = args[:hosts]
25
27
  @readonly = args[:readonly] ? true : false
26
28
  @timeout = args[:timeout]
29
+ @fail_timeout = args[:fail_timeout]
27
30
 
28
31
  reload
29
32
  end
@@ -32,7 +35,9 @@ class MogileFS::Client
32
35
  # Creates a new MogileFS::Backend.
33
36
 
34
37
  def reload
35
- @backend = MogileFS::Backend.new :hosts => @hosts, :timeout => @timeout
38
+ @backend = MogileFS::Backend.new(:hosts => @hosts,
39
+ :timeout => @timeout,
40
+ :fail_timeout => @fail_timeout)
36
41
  end
37
42
 
38
43
  ##
@@ -1,8 +1,7 @@
1
1
  # -*- encoding: binary -*-
2
2
  # here are internal implementation details, do not use them in your code
3
3
  require 'stringio'
4
- require 'uri'
5
- require 'mogilefs/chunker'
4
+ require 'mogilefs/new_file'
6
5
 
7
6
  ##
8
7
  # HTTPFile wraps up the new file operations for storing files onto an HTTP
@@ -12,49 +11,7 @@ require 'mogilefs/chunker'
12
11
  # create a new file using MogileFS::MogileFS.new_file.
13
12
  #
14
13
  class MogileFS::HTTPFile < StringIO
15
- class EmptyResponseError < MogileFS::Error; end
16
- class BadResponseError < MogileFS::Error; end
17
- class UnparseableResponseError < MogileFS::Error; end
18
- class NoStorageNodesError < MogileFS::Error
19
- def message; 'Unable to open socket to storage node'; end
20
- end
21
- class NonRetryableError < MogileFS::Error; end
22
-
23
- class HTTPSock < MogileFS::Socket
24
- attr_accessor :start
25
-
26
- # Increase timeout as we become more invested in uploading with
27
- # this socket. The server could be experiencing I/O delays
28
- # from large uploads because the sysadmin forgot to tune the
29
- # VM sysctls for handling large files.
30
- def write(buf)
31
- timed_write(buf, Time.now - @start + 5.0)
32
- end
33
- end
34
-
35
- # :stopdoc:
36
- MD5_TRAILER_NODES = {} # :nodoc: # EXPERIMENTAL
37
- class << self
38
- attr_accessor :response_timeout_cb
39
- end
40
-
41
- # temporary directories (nginx) may not be configured on the
42
- # same device, necessitating a time-consuming full file copy
43
- # instead of a quick rename(2)/link(2) operation
44
- @response_timeout_cb = lambda do |elapsed_time, bytes_uploaded|
45
- mbytes_uploaded = bytes_uploaded / (1024.0 * 1024.0)
46
- # assumes worst case is 10M/s on the remote storage disk
47
- t = mbytes_uploaded * 10 + elapsed_time
48
- t < 5 ? 5 : t
49
- end
50
- # :startdoc:
51
-
52
- ##
53
- # The URI this file will be stored to.
54
-
55
- attr_reader :uri
56
-
57
- attr_reader :devid
14
+ include MogileFS::NewFile::Common
58
15
 
59
16
  ##
60
17
  # The big_io name in case we have file > 256M
@@ -67,34 +24,50 @@ class MogileFS::HTTPFile < StringIO
67
24
  # Creates a new HTTPFile with MogileFS-specific data. Use
68
25
  # MogileFS::MogileFS#new_file instead of this method.
69
26
 
70
- def initialize(dests, content_length)
27
+ def initialize(dests, opts = nil)
71
28
  super ""
72
- @streaming_io = @big_io = @uri = @devid = @active = nil
29
+ @md5 = @streaming_io = @big_io = @active = nil
73
30
  @dests = dests
31
+ @opts = Integer === opts ? { :content_length => opts } : opts
74
32
  end
75
33
 
76
34
  def request_put(sock, uri, file_size, input = nil)
77
35
  host_with_port = "#{uri.host}:#{uri.port}"
78
- md5 = false
79
- if MD5_TRAILER_NODES[host_with_port]
36
+ clen = @opts[:content_length]
37
+ file_size ||= clen
38
+
39
+ content_md5 = @opts[:content_md5]
40
+ if String === content_md5
41
+ file_size or
42
+ raise ArgumentError,
43
+ ":content_length must be specified with :content_md5 String"
44
+ file_size = "#{file_size}\r\nContent-MD5: #{content_md5}"
45
+ elsif content_md5.respond_to?(:call) ||
46
+ :trailer == content_md5 ||
47
+ MD5_TRAILER_NODES[host_with_port]
80
48
  file_size = nil
81
- md5 = true
49
+ @md5 = Digest::MD5.new
82
50
  end
83
51
 
84
52
  if file_size
85
53
  sock.write("PUT #{uri.request_uri} HTTP/1.0\r\n" \
86
54
  "Content-Length: #{file_size}\r\n\r\n")
87
- input ? MogileFS.io.copy_stream(@active = input, sock) : yield(sock)
55
+ rv = input ? MogileFS.io.copy_stream(@active = input, sock) : yield(sock)
88
56
  else
89
- trailers = md5 ? "Trailer: Content-MD5\r\n" : ""
57
+ trailers = @md5 ? "Trailer: Content-MD5\r\n" : ""
90
58
  sock.write("PUT #{uri.request_uri} HTTP/1.1\r\n" \
91
59
  "Host: #{host_with_port}\r\n#{trailers}" \
92
60
  "Transfer-Encoding: chunked\r\n\r\n")
93
- tmp = MogileFS::Chunker.new(sock, md5)
61
+ tmp = MogileFS::Chunker.new(sock, @md5, content_md5)
94
62
  rv = input ? MogileFS.io.copy_stream(@active = input, tmp) : yield(tmp)
95
63
  tmp.flush
96
- rv
97
64
  end
65
+
66
+ if clen && clen != rv
67
+ raise MogileFS::SizeMismatchError,
68
+ ":content_length expected: #{clen.inspect}, actual: #{rv.inspect}"
69
+ end
70
+ rv
98
71
  end
99
72
 
100
73
  def put_streaming_io(sock, uri) # unlikely to be used
@@ -121,51 +94,39 @@ class MogileFS::HTTPFile < StringIO
121
94
  # Writes an HTTP PUT request to +sock+ to upload the file and
122
95
  # returns file size if the socket finished writing
123
96
  def upload(devid, uri) # :nodoc:
124
- start = Time.now
125
- sock = HTTPSock.tcp(uri.host, uri.port)
126
- sock.start = start
97
+ sock = MogileFS::Socket.tcp(uri.host, uri.port)
98
+ set_socket_options(sock)
127
99
  file_size = length
128
100
 
129
101
  if @streaming_io
130
102
  file_size = put_streaming_io(sock, uri)
131
103
  elsif @big_io
132
- if String === @big_io || @big_io.respond_to?(:to_path)
104
+ stat = file = size = nil
105
+ if @big_io.respond_to?(:stat)
106
+ stat = @big_io.stat
107
+ elsif String === @big_io || @big_io.respond_to?(:to_path)
133
108
  file = File.open(@big_io)
134
109
  stat = file.stat
135
- file_size = request_put(sock, uri, stat.file? ? stat.size : nil, file)
136
- else
137
- size = nil
138
- if @big_io.respond_to?(:stat)
139
- stat = @big_io.stat
140
- size = stat.size if stat.file?
141
- elsif @big_io.respond_to?(:size)
142
- size = @big_io.size
143
- end
144
- file_size = request_put(sock, uri, size, @big_io)
110
+ elsif @big_io.respond_to?(:size)
111
+ size = @big_io.size
112
+ end
113
+ if stat && stat.file?
114
+ size ||= stat.size
115
+ file ||= @big_io.to_io if @big_io.respond_to?(:to_io)
145
116
  end
117
+ file_size = request_put(sock, uri, size, file || @big_io)
146
118
  else
147
119
  rewind
148
120
  request_put(sock, uri, file_size, self)
149
121
  end
150
122
 
151
- tout = self.class.response_timeout_cb.call(Time.now - start, file_size)
152
-
153
- case line = sock.timed_read(23, "", tout)
154
- when %r{^HTTP/\d\.\d\s+(2\d\d)\s} # success!
155
- file_size
156
- when nil
157
- raise EmptyResponseError, 'Unable to read response line from server'
158
- when %r{^HTTP/\d\.\d\s+(\d+)}
159
- raise BadResponseError, "HTTP response status from upload: #$1"
160
- else
161
- raise UnparseableResponseError,
162
- "Response line not understood: #{line.inspect}"
163
- end
164
- rescue => err
123
+ read_response(sock) # raises on errors
124
+ file_size
125
+ rescue SystemCallError, RetryableError => err
165
126
  rewind_or_raise!(uri, err)
166
127
  raise
167
128
  ensure
168
- file.close if file
129
+ file.close if file && @big_io != file
169
130
  sock.close if sock
170
131
  end
171
132
 
@@ -175,11 +136,8 @@ class MogileFS::HTTPFile < StringIO
175
136
  begin
176
137
  uri = URI.parse(path)
177
138
  bytes_uploaded = upload(devid, uri)
178
- @devid, @uri = devid, uri
179
- return bytes_uploaded
180
- rescue NonRetryableError
181
- raise
182
- rescue => e
139
+ return create_close(devid, uri, bytes_uploaded)
140
+ rescue SystemCallError, RetryableError => e
183
141
  errors ||= []
184
142
  errors << "#{path} - #{e.message} (#{e.class})"
185
143
  end
@@ -188,4 +146,9 @@ class MogileFS::HTTPFile < StringIO
188
146
  raise NoStorageNodesError,
189
147
  "all paths failed with PUT: #{errors.join(', ')}", []
190
148
  end
149
+
150
+ def close
151
+ commit
152
+ super
153
+ end
191
154
  end
@@ -34,15 +34,41 @@ class MogileFS::MogileFS < MogileFS::Client
34
34
  # The domain of keys for this MogileFS client.
35
35
  attr_accessor :domain
36
36
 
37
- # The timeout for get_file_data. Defaults to five seconds.
37
+ # The timeout for get_file_data (per-read() system call).
38
+ # Defaults to five seconds.
38
39
  attr_accessor :get_file_data_timeout
39
40
 
41
+ # The maximum allowed time for creating a new_file. Defaults to 1 hour.
42
+ attr_accessor :new_file_max_time
43
+
40
44
  # Creates a new MogileFS::MogileFS instance. +args+ must include a key
41
45
  # :domain specifying the domain of this client.
46
+ #
47
+ # Optional parameters for +args+:
48
+ #
49
+ # [:get_file_data_timeout => Numeric]
50
+ #
51
+ # See get_file_data_timeout
52
+ #
53
+ # [:new_file_max_time => Numeric]
54
+ #
55
+ # See new_file_max_time
56
+ #
57
+ # [:fail_timeout => Numeric]
58
+ #
59
+ # Delay before retrying a failed tracker backends.
60
+ # Defaults to 5 seconds.
61
+ #
62
+ # [:timeout => Numeric]
63
+ #
64
+ # Timeout for tracker backend responses.
65
+ # Defaults to 3 seconds.
66
+ #
42
67
  def initialize(args = {})
43
68
  @domain = args[:domain]
44
69
 
45
- @get_file_data_timeout = 5
70
+ @get_file_data_timeout = args[:get_file_data_timeout] || 5
71
+ @new_file_max_time = args[:new_file_max_time] || 3600.0
46
72
 
47
73
  raise ArgumentError, "you must specify a domain" unless @domain
48
74
 
@@ -113,11 +139,8 @@ class MogileFS::MogileFS < MogileFS::Client
113
139
 
114
140
  # Returns +true+ if +key+ exists, +false+ if not
115
141
  def exist?(key)
116
- rv = nil
117
- args = { :key => key, :domain => @domain }
118
- @backend.pipeline_dispatch(:get_paths, args) { |x| rv = (Hash === x) }
119
- @backend.pipeline_wait(1)
120
- rv
142
+ args = { :key => key, :domain => @domain , :ruby_no_raise => true}
143
+ Hash === @backend.get_paths(args)
121
144
  end
122
145
 
123
146
  # Get the URIs for +key+ (paths) as URI::HTTP objects
@@ -125,20 +148,45 @@ class MogileFS::MogileFS < MogileFS::Client
125
148
  get_paths(key, *args).map! { |path| URI.parse(path) }
126
149
  end
127
150
 
128
- # Creates a new file +key+ in +klass+. +bytes+ is currently unused.
129
- # Consider using store_file instead of this method for large files.
130
- # This requires a block passed to it and operates like File.open.
131
- # This atomically replaces existing data stored as +key+ when
132
- def new_file(key, klass = nil, bytes = 0) # :yields: file
151
+ # Creates a new file +key+ in the domain of this object.
152
+ #
153
+ # +bytes+ is the expected size of the file if known in advance
154
+ #
155
+ # It operates like File.open(..., "w") and may take an optional
156
+ # block, yielding an IO-like object with support for the methods
157
+ # documented in MogileFS::NewFile::Writer.
158
+ #
159
+ # This atomically replaces existing data stored as +key+
160
+ # when the block exits or when the returned object is closed.
161
+ #
162
+ # +args+ may contain the following options:
163
+ #
164
+ # [:content_length => Integer]
165
+ #
166
+ # This has the same effect as the (deprecated) +bytes+ parameter.
167
+ #
168
+ # [ :largefile => :stream, :content_range or :tempfile ]
169
+ #
170
+ # See MogileFS::NewFile for more information on this
171
+ #
172
+ # [ :class => String]
173
+ #
174
+ # The MogileFS storage class of the object.
175
+ def new_file(key, args = nil, bytes = nil) # :yields: file
133
176
  raise MogileFS::ReadOnlyError if readonly?
134
- opts = { :domain => @domain, :key => key, :multi_dest => 1 }
135
- opts[:class] = klass if klass && klass != "default"
177
+ opts = { :key => key, :multi_dest => 1 }
178
+ case args
179
+ when Hash
180
+ opts[:domain] = args[:domain]
181
+ klass = args[:class] and "default" != klass and opts[:class] = klass
182
+ when String
183
+ opts[:class] = args if "default" != args
184
+ end
185
+ opts[:domain] ||= @domain
136
186
  res = @backend.create_open(opts)
137
187
 
138
188
  dests = if dev_count = res['dev_count'] # multi_dest succeeded
139
- (1..dev_count.to_i).map do |i|
140
- [res["devid_#{i}"], res["path_#{i}"]]
141
- end
189
+ (1..dev_count.to_i).map { |i| [res["devid_#{i}"], res["path_#{i}"]] }
142
190
  else # single destination returned
143
191
  # 0x0040: d0e4 4f4b 2064 6576 6964 3d31 2666 6964 ..OK.devid=1&fid
144
192
  # 0x0050: 3d33 2670 6174 683d 6874 7470 3a2f 2f31 =3&path=http://1
@@ -149,19 +197,23 @@ class MogileFS::MogileFS < MogileFS::Client
149
197
  [[res['devid'], res['path']]]
150
198
  end
151
199
 
200
+ opts.merge!(args) if Hash === args
201
+ opts[:backend] = @backend
202
+ opts[:fid] = res['fid']
203
+ opts[:content_length] ||= bytes if bytes
204
+ opts[:new_file_max_time] ||= @new_file_max_time
205
+ opts[:start_time] = Time.now
206
+
152
207
  case (dests[0][1] rescue nil)
153
- when /^http:\/\// then
154
- http_file = MogileFS::HTTPFile.new(dests, bytes)
155
- yield http_file
156
- rv = http_file.commit
157
- @backend.create_close(:fid => res['fid'],
158
- :devid => http_file.devid,
159
- :domain => @domain,
160
- :key => key,
161
- :path => http_file.uri.to_s,
162
- :size => rv)
163
- rv
164
- when nil, '' then
208
+ when %r{\Ahttp://}
209
+ http_file = MogileFS::NewFile.new(dests, opts)
210
+ if block_given?
211
+ yield http_file
212
+ return http_file.commit # calls create_close
213
+ else
214
+ return http_file
215
+ end
216
+ when nil, ''
165
217
  raise MogileFS::EmptyPathError,
166
218
  "Empty path for mogile upload res=#{res.inspect}"
167
219
  else
@@ -174,18 +226,20 @@ class MogileFS::MogileFS < MogileFS::Client
174
226
  # either a path name (String or Pathname object) or an IO-like object that
175
227
  # responds to #read or #readpartial. Returns size of +file+ stored.
176
228
  # This atomically replaces existing data stored as +key+
177
- def store_file(key, klass, file)
229
+ def store_file(key, klass, file, opts = nil)
178
230
  raise MogileFS::ReadOnlyError if readonly?
231
+ (opts ||= {})[:class] = klass if String === klass
179
232
 
180
- new_file(key, klass) { |mfp| mfp.big_io = file }
233
+ new_file(key, opts) { |mfp| mfp.big_io = file }
181
234
  end
182
235
 
183
236
  # Stores +content+ into +key+ in class +klass+, where +content+ is a String
184
237
  # This atomically replaces existing data stored as +key+
185
- def store_content(key, klass, content)
238
+ def store_content(key, klass, content, opts = nil)
186
239
  raise MogileFS::ReadOnlyError if readonly?
240
+ (opts ||= {})[:class] = klass if String === klass
187
241
 
188
- new_file key, klass do |mfp|
242
+ new_file(key, opts) do |mfp|
189
243
  if content.is_a?(MogileFS::Util::StoreContent)
190
244
  mfp.streaming_io = content
191
245
  else
@@ -236,12 +290,10 @@ class MogileFS::MogileFS < MogileFS::Client
236
290
  @backend.respond_to?(:_list_keys) and
237
291
  return @backend._list_keys(domain, prefix, after, limit, &block)
238
292
 
239
- begin
240
- res = @backend.list_keys(:domain => domain, :prefix => prefix,
241
- :after => after, :limit => limit)
242
- rescue MogileFS::Backend::NoneMatchError
243
- return
244
- end
293
+ res = @backend.list_keys(:domain => domain, :prefix => prefix,
294
+ :after => after, :limit => limit,
295
+ :ruby_no_raise => true)
296
+ MogileFS::Backend::NoneMatchError === res and return
245
297
 
246
298
  keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
247
299
  if block
@@ -0,0 +1,78 @@
1
+ # -*- encoding: binary -*-
2
+ #
3
+ # The MogileFS::MogileFS#new_file method is enhanced in v3.1.0+
4
+ # to support the :largefile parameter. While we have always
5
+ # supported large files via the "store_file" method, streaming
6
+ # large amounts of content of an unknown length required the use
7
+ # of awkward APIs.
8
+ #
9
+ # It is possible to stream large content of known length any WebDAV server.
10
+ # One example of this is for mirroring a large file from an existing HTTP
11
+ # server to \MogileFS without letting it hit the local filesystem.
12
+ #
13
+ # uri = URI('http://example.com/large_file')
14
+ # Net::HTTP.start(uri.host, uri.port) do |http|
15
+ # req = Net::HTTP::Get.new(uri.request_uri)
16
+ #
17
+ # http.request(req) do |response|
18
+ # if len = response.content_length
19
+ # io = mg.new_file('key', :largefile => true, :content_length => len)
20
+ # else
21
+ # warn "trying to upload with Transfer-Encoding: chunked"
22
+ # warn "this is not supported by all WebDAV servers"
23
+ # io = mg.new_file('key', :largefile => :stream)
24
+ # end
25
+ # response.read_body { |buf| io.write(buf) }
26
+ # io.close
27
+ # end
28
+ # end
29
+ #
30
+ # If your WebDAV servers have chunked PUT support (e.g. Perlbal), you can
31
+ # stream a file of unknown length using "Transfer-Encoding: chunked".
32
+ #
33
+ # nf = mg.new_file("key", :largefile => :stream)
34
+ # nf.write "hello"
35
+ # nf.write ...
36
+ # nf.close
37
+ #
38
+ # If your WebDAV server has partial PUT support (e.g Apache), you can
39
+ # you can use multiple PUT requests with "Content-Range" support.
40
+ # This method is slower than Transfer-Encoding: chunked.
41
+ #
42
+ # nf = mg.new_file("key", :largefile => :content_range)
43
+ # nf.write "hello"
44
+ # nf.write ...
45
+ # nf.close
46
+ #
47
+ # Finally, if your WebDAV servers does not support either partial nor
48
+ # nor chunked PUTs, you must buffer a large file of unknown length
49
+ # using a Tempfile:
50
+ #
51
+ # nf = mg.new_file("key", :largefile => :tempfile)
52
+ # nf.write "hello"
53
+ # nf.write ...
54
+ # nf.close
55
+ #
56
+ module MogileFS::NewFile
57
+
58
+ # avoiding autoload for new code since it's going away in Ruby...
59
+ def self.new(dests, opts) # :nodoc:
60
+ largefile = opts[:largefile]
61
+ largefile = :stream if largefile && opts[:content_length]
62
+ require "mogilefs/new_file/#{largefile}" if Symbol === largefile
63
+ case largefile
64
+ when nil, false
65
+ MogileFS::HTTPFile
66
+ when :stream
67
+ Stream
68
+ when :content_range
69
+ ContentRange
70
+ when :tempfile
71
+ Tempfile
72
+ else
73
+ raise ArgumentError, "largefile: #{largefile.inspect} not understood"
74
+ end.new(dests, opts)
75
+ end
76
+ end
77
+
78
+ require 'mogilefs/new_file/common'