mogilefs-client 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +32 -0
- data/Manifest.txt +18 -0
- data/README +9 -0
- data/Rakefile +56 -0
- data/lib/mogilefs.rb +24 -0
- data/lib/mogilefs/admin.rb +274 -0
- data/lib/mogilefs/backend.rb +219 -0
- data/lib/mogilefs/client.rb +64 -0
- data/lib/mogilefs/httpfile.rb +144 -0
- data/lib/mogilefs/mogilefs.rb +214 -0
- data/lib/mogilefs/nfsfile.rb +81 -0
- data/lib/mogilefs/pool.rb +50 -0
- data/test/setup.rb +55 -0
- data/test/test_admin.rb +37 -0
- data/test/test_backend.rb +208 -0
- data/test/test_client.rb +49 -0
- data/test/test_mogilefs.rb +123 -0
- data/test/test_pool.rb +98 -0
- metadata +63 -0
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'mogilefs/backend'
|
2
|
+
|
3
|
+
##
|
4
|
+
# MogileFS::Client is the MogileFS client base class. Concrete clients like
|
5
|
+
# MogileFS::MogileFS and MogileFS::Admin are implemented atop this one to do
|
6
|
+
# real work.
|
7
|
+
|
8
|
+
class MogileFS::Client
|
9
|
+
|
10
|
+
##
|
11
|
+
# The backend connection for this client
|
12
|
+
|
13
|
+
attr_reader :backend
|
14
|
+
|
15
|
+
attr_accessor :hosts if defined? $TESTING
|
16
|
+
|
17
|
+
##
|
18
|
+
# Creates a new Client. See MogileFS::Backend#initialize for how to specify
|
19
|
+
# hosts. If :readonly is set to true, the client will not modify anything
|
20
|
+
# on the server.
|
21
|
+
#
|
22
|
+
# MogileFS::Client.new :hosts => ['kaa:6001', 'ziz:6001'], :readonly => true
|
23
|
+
|
24
|
+
def initialize(args)
|
25
|
+
@hosts = args[:hosts]
|
26
|
+
@readonly = args[:readonly] ? true : false
|
27
|
+
|
28
|
+
reload
|
29
|
+
end
|
30
|
+
|
31
|
+
##
|
32
|
+
# Creates a new MogileFS::Backend.
|
33
|
+
|
34
|
+
def reload
|
35
|
+
@backend = MogileFS::Backend.new :hosts => @hosts
|
36
|
+
end
|
37
|
+
|
38
|
+
##
|
39
|
+
# The last error reported by the backend.
|
40
|
+
#--
|
41
|
+
# TODO use Exceptions
|
42
|
+
|
43
|
+
def err
|
44
|
+
@backend.lasterr
|
45
|
+
end
|
46
|
+
|
47
|
+
##
|
48
|
+
# The last error message reported by the backend.
|
49
|
+
#--
|
50
|
+
# TODO use Exceptions
|
51
|
+
|
52
|
+
def errstr
|
53
|
+
@backend.lasterrstr
|
54
|
+
end
|
55
|
+
|
56
|
+
##
|
57
|
+
# Is this a read-only client?
|
58
|
+
|
59
|
+
def readonly?
|
60
|
+
return @readonly
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
|
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'fcntl'
|
2
|
+
require 'socket'
|
3
|
+
require 'stringio'
|
4
|
+
require 'uri'
|
5
|
+
require 'mogilefs/backend'
|
6
|
+
|
7
|
+
##
|
8
|
+
# HTTPFile wraps up the new file operations for storing files onto an HTTP
|
9
|
+
# storage node.
|
10
|
+
#
|
11
|
+
# You really don't want to create an HTTPFile by hand. Instead you want to
|
12
|
+
# create a new file using MogileFS::MogileFS.new_file.
|
13
|
+
#
|
14
|
+
# WARNING! HTTP mode is completely untested as I cannot make it work on
|
15
|
+
# FreeBSD. Please send patches/tests if you find bugs.
|
16
|
+
#--
|
17
|
+
# TODO dup'd content in MogileFS::NFSFile
|
18
|
+
|
19
|
+
class MogileFS::HTTPFile < StringIO
|
20
|
+
|
21
|
+
##
|
22
|
+
# The path this file will be stored to.
|
23
|
+
|
24
|
+
attr_reader :path
|
25
|
+
|
26
|
+
##
|
27
|
+
# The key for this file. This key won't represent a real file until you've
|
28
|
+
# called #close.
|
29
|
+
|
30
|
+
attr_reader :key
|
31
|
+
|
32
|
+
##
|
33
|
+
# The class of this file.
|
34
|
+
|
35
|
+
attr_reader :class
|
36
|
+
|
37
|
+
##
|
38
|
+
# Works like File.open. Use MogileFS::MogileFS#new_file instead of this
|
39
|
+
# method.
|
40
|
+
|
41
|
+
def self.open(*args)
|
42
|
+
fp = new(*args)
|
43
|
+
|
44
|
+
return fp unless block_given?
|
45
|
+
|
46
|
+
begin
|
47
|
+
yield fp
|
48
|
+
ensure
|
49
|
+
fp.close
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
##
|
54
|
+
# Creates a new HTTPFile with MogileFS-specific data. Use
|
55
|
+
# MogileFS::MogileFS#new_file instead of this method.
|
56
|
+
|
57
|
+
def initialize(mg, fid, path, devid, klass, key, dests, content_length)
|
58
|
+
super ''
|
59
|
+
@mg = mg
|
60
|
+
@fid = fid
|
61
|
+
@path = path
|
62
|
+
@devid = devid
|
63
|
+
@klass = klass
|
64
|
+
@key = key
|
65
|
+
|
66
|
+
@dests = dests.map { |(_,u)| URI.parse u }
|
67
|
+
@tried = {}
|
68
|
+
|
69
|
+
@socket = nil
|
70
|
+
end
|
71
|
+
|
72
|
+
##
|
73
|
+
# Closes the file handle and marks it as closed in MogileFS.
|
74
|
+
|
75
|
+
def close
|
76
|
+
connect_socket
|
77
|
+
|
78
|
+
@socket.write "PUT #{@path.request_uri} HTTP/1.0\r\nContent-length: #{length}\r\n\r\n#{string}"
|
79
|
+
|
80
|
+
if connected? then
|
81
|
+
line = @socket.gets
|
82
|
+
raise 'Unable to read response line from server' if line.nil?
|
83
|
+
|
84
|
+
if line =~ %r%^HTTP/\d+\.\d+\s+(\d+)% then
|
85
|
+
status = Integer $1
|
86
|
+
case status
|
87
|
+
when 200..299 then # success!
|
88
|
+
else
|
89
|
+
found_header = false
|
90
|
+
body = []
|
91
|
+
line = @socket.gets
|
92
|
+
until line.nil? do
|
93
|
+
line.strip
|
94
|
+
found_header = true if line.nil?
|
95
|
+
next unless found_header
|
96
|
+
body << " #{line}"
|
97
|
+
end
|
98
|
+
body = body[0, 512] if body.length > 512
|
99
|
+
raise "HTTP response status from upload: #{body}"
|
100
|
+
end
|
101
|
+
else
|
102
|
+
raise "Response line not understood: #{line}"
|
103
|
+
end
|
104
|
+
@socket.close
|
105
|
+
end
|
106
|
+
|
107
|
+
@mg.backend.create_close(:fid => @fid, :devid => @devid,
|
108
|
+
:domain => @mg.domain, :key => @key,
|
109
|
+
:path => @path, :size => length)
|
110
|
+
return nil
|
111
|
+
end
|
112
|
+
|
113
|
+
private
|
114
|
+
|
115
|
+
def connected?
|
116
|
+
return !(@socket.nil? or @socket.closed?)
|
117
|
+
end
|
118
|
+
|
119
|
+
def connect_socket
|
120
|
+
return @socket if connected?
|
121
|
+
|
122
|
+
next_path
|
123
|
+
|
124
|
+
if @path.nil? then
|
125
|
+
@tried.clear
|
126
|
+
next_path
|
127
|
+
raise 'Unable to open socket to storage node' if @path.nil?
|
128
|
+
end
|
129
|
+
|
130
|
+
@socket = TCPSocket.new @path.host, @path.port
|
131
|
+
end
|
132
|
+
|
133
|
+
def next_path
|
134
|
+
@path = nil
|
135
|
+
@dests.each do |dest|
|
136
|
+
unless @tried.include? dest then
|
137
|
+
@path = dest
|
138
|
+
return
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
end
|
144
|
+
|
@@ -0,0 +1,214 @@
|
|
1
|
+
require 'open-uri'
|
2
|
+
require 'timeout'
|
3
|
+
|
4
|
+
require 'mogilefs/client'
|
5
|
+
require 'mogilefs/nfsfile'
|
6
|
+
|
7
|
+
##
|
8
|
+
# Timeout error class.
|
9
|
+
|
10
|
+
class MogileFS::Timeout < Timeout::Error; end
|
11
|
+
|
12
|
+
##
|
13
|
+
# MogileFS File manipulation client.
|
14
|
+
|
15
|
+
class MogileFS::MogileFS < MogileFS::Client
|
16
|
+
|
17
|
+
##
|
18
|
+
# The path to the local MogileFS mount point if you are using NFS mode.
|
19
|
+
|
20
|
+
attr_reader :root
|
21
|
+
|
22
|
+
##
|
23
|
+
# The domain of keys for this MogileFS client.
|
24
|
+
|
25
|
+
attr_reader :domain
|
26
|
+
|
27
|
+
##
|
28
|
+
# Creates a new MogileFS::MogileFS instance. +args+ must include a key
|
29
|
+
# :domain specifying the domain of this client. A key :root will be used to
|
30
|
+
# specify the root of the NFS file system.
|
31
|
+
|
32
|
+
def initialize(args = {})
|
33
|
+
@domain = args[:domain]
|
34
|
+
@root = args[:root]
|
35
|
+
|
36
|
+
raise ArgumentError, "you must specify a domain" unless @domain
|
37
|
+
|
38
|
+
super
|
39
|
+
end
|
40
|
+
|
41
|
+
##
|
42
|
+
# Retrieves the contents of +key+.
|
43
|
+
|
44
|
+
def get_file_data(key)
|
45
|
+
get_paths(key).each do |path|
|
46
|
+
next unless path
|
47
|
+
case path
|
48
|
+
when /^http:\/\// then
|
49
|
+
begin
|
50
|
+
puts "path: #{path}"
|
51
|
+
path = URI.parse path
|
52
|
+
data = timeout(5, MogileFS::Timeout) { path.read }
|
53
|
+
puts "data: #{data}"
|
54
|
+
rescue MogileFS::Timeout
|
55
|
+
next
|
56
|
+
end
|
57
|
+
else
|
58
|
+
next unless File.exist? path
|
59
|
+
return File.read(path)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
return nil
|
64
|
+
end
|
65
|
+
|
66
|
+
##
|
67
|
+
# Get the paths for +key+.
|
68
|
+
|
69
|
+
def get_paths(key, noverify = true, zone = nil)
|
70
|
+
noverify = noverify ? 1 : 0
|
71
|
+
res = @backend.get_paths(:domain => @domain, :key => key,
|
72
|
+
:noverify => noverify, :zone => zone)
|
73
|
+
|
74
|
+
return nil if res.nil? and @backend.lasterr == 'unknown_key'
|
75
|
+
paths = (1..res['paths'].to_i).map { |i| res["path#{i}"] }
|
76
|
+
return paths if paths.empty?
|
77
|
+
return paths if paths.first =~ /^http:\/\//
|
78
|
+
return paths.map { |path| File.join @root, path }
|
79
|
+
end
|
80
|
+
|
81
|
+
##
|
82
|
+
# Creates a new file +key+ in +klass+. +bytes+ is currently unused.
|
83
|
+
#
|
84
|
+
# The +block+ operates like File.open.
|
85
|
+
|
86
|
+
def new_file(key, klass, bytes = 0, &block) # :yields: file
|
87
|
+
raise 'readonly mogilefs' if readonly?
|
88
|
+
|
89
|
+
res = @backend.create_open(:domain => @domain, :class => klass,
|
90
|
+
:key => key, :multi_dest => 1)
|
91
|
+
|
92
|
+
raise "#{@backend.lasterr}: #{@backend.lasterrstr}" if res.nil? # HACK
|
93
|
+
|
94
|
+
dests = nil
|
95
|
+
|
96
|
+
if res.include? 'dev_count' then # HACK HUH?
|
97
|
+
dests = (1..res['dev_count'].to_i).map do |i|
|
98
|
+
[res["devid_#{i}"], res["path_#{i}"]]
|
99
|
+
end
|
100
|
+
else
|
101
|
+
# 0x0040: d0e4 4f4b 2064 6576 6964 3d31 2666 6964 ..OK.devid=1&fid
|
102
|
+
# 0x0050: 3d33 2670 6174 683d 6874 7470 3a2f 2f31 =3&path=http://1
|
103
|
+
# 0x0060: 3932 2e31 3638 2e31 2e37 323a 3735 3030 92.168.1.72:7500
|
104
|
+
# 0x0070: 2f64 6576 312f 302f 3030 302f 3030 302f /dev1/0/000/000/
|
105
|
+
# 0x0080: 3030 3030 3030 3030 3033 2e66 6964 0d0a 0000000003.fid..
|
106
|
+
|
107
|
+
dests = [[res['devid'], res['path']]]
|
108
|
+
end
|
109
|
+
|
110
|
+
dest = dests.first
|
111
|
+
devid, path = dest
|
112
|
+
|
113
|
+
case path
|
114
|
+
when /^http:\/\// then
|
115
|
+
MogileFS::HTTPFile.open(self, res['fid'], path, devid, klass, key,
|
116
|
+
dests, bytes, &block)
|
117
|
+
else
|
118
|
+
MogileFS::NFSFile.open(self, res['fid'], path, devid, klass, key, &block)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
##
|
123
|
+
# Copies the contents of +file+ into +key+ in class +klass+. +file+ can be
|
124
|
+
# either a file name or an object that responds to #read.
|
125
|
+
|
126
|
+
def store_file(key, klass, file)
|
127
|
+
raise 'readonly mogilefs' if readonly?
|
128
|
+
|
129
|
+
new_file key, klass do |mfp|
|
130
|
+
if file.respond_to? :read then
|
131
|
+
return copy(file, mfp)
|
132
|
+
else
|
133
|
+
return File.open(file) { |fp| copy(fp, mfp) }
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
##
|
139
|
+
# Stores +content+ into +key+ in class +klass+.
|
140
|
+
|
141
|
+
def store_content(key, klass, content)
|
142
|
+
raise 'readonly mogilefs' if readonly?
|
143
|
+
|
144
|
+
new_file key, klass do |mfp|
|
145
|
+
mfp << content
|
146
|
+
end
|
147
|
+
|
148
|
+
return content.length
|
149
|
+
end
|
150
|
+
|
151
|
+
##
|
152
|
+
# Removes +key+.
|
153
|
+
|
154
|
+
def delete(key)
|
155
|
+
raise 'readonly mogilefs' if readonly?
|
156
|
+
|
157
|
+
res = @backend.delete :domain => @domain, :key => key
|
158
|
+
|
159
|
+
if res.nil? and @backend.lasterr != 'unknown_key' then
|
160
|
+
raise "unable to delete #{key}: #{@backend.lasterr}"
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
##
|
165
|
+
# Sleeps +duration+.
|
166
|
+
|
167
|
+
def sleep(duration)
|
168
|
+
@backend.sleep :duration => duration
|
169
|
+
end
|
170
|
+
|
171
|
+
##
|
172
|
+
# Renames a key +from+ to key +to+.
|
173
|
+
|
174
|
+
def rename(from, to)
|
175
|
+
raise 'readonly mogilefs' if readonly?
|
176
|
+
|
177
|
+
res = @backend.rename :domain => @domain, :from_key => from, :to_key => to
|
178
|
+
|
179
|
+
if res.nil? and @backend.lasterr != 'unknown_key' then
|
180
|
+
raise "unable to rename #{from_key} to #{to_key}: #{@backend.lasterr}"
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
##
|
185
|
+
# Lists keys starting with +prefix+ follwing +after+ up to +limit+. If
|
186
|
+
# +after+ is nil the list starts at the beginning.
|
187
|
+
|
188
|
+
def list_keys(prefix, after = nil, limit = 1000)
|
189
|
+
res = @backend.list_keys(:domain => domain, :prefix => prefix,
|
190
|
+
:after => after, :limit => limit)
|
191
|
+
|
192
|
+
return nil if res.nil?
|
193
|
+
|
194
|
+
keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
|
195
|
+
|
196
|
+
return keys, res['next_after']
|
197
|
+
end
|
198
|
+
|
199
|
+
private
|
200
|
+
|
201
|
+
def copy(from, to) # HACK use FileUtils
|
202
|
+
bytes = 0
|
203
|
+
|
204
|
+
until from.eof? do
|
205
|
+
chunk = from.read 8192
|
206
|
+
to.write chunk
|
207
|
+
bytes += chunk.length
|
208
|
+
end
|
209
|
+
|
210
|
+
return bytes
|
211
|
+
end
|
212
|
+
|
213
|
+
end
|
214
|
+
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'mogilefs/backend'
|
2
|
+
|
3
|
+
##
|
4
|
+
# NFSFile wraps up the new file operations for storing files onto an NFS
|
5
|
+
# storage node.
|
6
|
+
#
|
7
|
+
# You really don't want to create an NFSFile by hand. Instead you want to
|
8
|
+
# create a new file using MogileFS::MogileFS.new_file.
|
9
|
+
|
10
|
+
class MogileFS::NFSFile < File
|
11
|
+
|
12
|
+
##
|
13
|
+
# The path of this file not including the local mount point.
|
14
|
+
|
15
|
+
attr_reader :path
|
16
|
+
|
17
|
+
##
|
18
|
+
# The key for this file. This key won't represent a real file until you've
|
19
|
+
# called #close.
|
20
|
+
|
21
|
+
attr_reader :key
|
22
|
+
|
23
|
+
##
|
24
|
+
# The class of this file.
|
25
|
+
|
26
|
+
attr_reader :class
|
27
|
+
|
28
|
+
class << self
|
29
|
+
|
30
|
+
##
|
31
|
+
# Wraps up File.new with MogileFS-specific data. Use
|
32
|
+
# MogileFS::MogileFS#new_file instead of this method.
|
33
|
+
|
34
|
+
def new(mg, fid, path, devid, klass, key)
|
35
|
+
fp = super join(mg.root, path), 'w+'
|
36
|
+
fp.send :setup, mg, fid, path, devid, klass, key
|
37
|
+
return fp
|
38
|
+
end
|
39
|
+
|
40
|
+
##
|
41
|
+
# Wraps up File.open with MogileFS-specific data. Use
|
42
|
+
# MogileFS::MogileFS#new_file instead of this method.
|
43
|
+
|
44
|
+
def open(mg, fid, path, devid, klass, key, &block)
|
45
|
+
fp = new mg, fid, path, devid, klass, key
|
46
|
+
|
47
|
+
return fp if block.nil?
|
48
|
+
|
49
|
+
begin
|
50
|
+
yield fp
|
51
|
+
ensure
|
52
|
+
fp.close
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
##
|
59
|
+
# Closes the file handle and marks it as closed in MogileFS.
|
60
|
+
|
61
|
+
def close
|
62
|
+
super
|
63
|
+
@mg.backend.create_close(:fid => @fid, :devid => @devid,
|
64
|
+
:domain => @mg.domain, :key => @key,
|
65
|
+
:path => @path)
|
66
|
+
return nil
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
|
71
|
+
def setup(mg, fid, path, devid, klass, key)
|
72
|
+
@mg = mg
|
73
|
+
@fid = fid
|
74
|
+
@path = path
|
75
|
+
@devid = devid
|
76
|
+
@klass = klass
|
77
|
+
@key = key
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
81
|
+
|