mogilefs-client 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,64 @@
1
+ require 'mogilefs/backend'
2
+
3
+ ##
4
+ # MogileFS::Client is the MogileFS client base class. Concrete clients like
5
+ # MogileFS::MogileFS and MogileFS::Admin are implemented atop this one to do
6
+ # real work.
7
+
8
+ class MogileFS::Client
9
+
10
+ ##
11
+ # The backend connection for this client
12
+
13
+ attr_reader :backend
14
+
15
+ attr_accessor :hosts if defined? $TESTING
16
+
17
+ ##
18
+ # Creates a new Client. See MogileFS::Backend#initialize for how to specify
19
+ # hosts. If :readonly is set to true, the client will not modify anything
20
+ # on the server.
21
+ #
22
+ # MogileFS::Client.new :hosts => ['kaa:6001', 'ziz:6001'], :readonly => true
23
+
24
+ def initialize(args)
25
+ @hosts = args[:hosts]
26
+ @readonly = args[:readonly] ? true : false
27
+
28
+ reload
29
+ end
30
+
31
+ ##
32
+ # Creates a new MogileFS::Backend.
33
+
34
+ def reload
35
+ @backend = MogileFS::Backend.new :hosts => @hosts
36
+ end
37
+
38
+ ##
39
+ # The last error reported by the backend.
40
+ #--
41
+ # TODO use Exceptions
42
+
43
+ def err
44
+ @backend.lasterr
45
+ end
46
+
47
+ ##
48
+ # The last error message reported by the backend.
49
+ #--
50
+ # TODO use Exceptions
51
+
52
+ def errstr
53
+ @backend.lasterrstr
54
+ end
55
+
56
+ ##
57
+ # Is this a read-only client?
58
+
59
+ def readonly?
60
+ return @readonly
61
+ end
62
+
63
+ end
64
+
@@ -0,0 +1,144 @@
1
+ require 'fcntl'
2
+ require 'socket'
3
+ require 'stringio'
4
+ require 'uri'
5
+ require 'mogilefs/backend'
6
+
7
+ ##
8
+ # HTTPFile wraps up the new file operations for storing files onto an HTTP
9
+ # storage node.
10
+ #
11
+ # You really don't want to create an HTTPFile by hand. Instead you want to
12
+ # create a new file using MogileFS::MogileFS.new_file.
13
+ #
14
+ # WARNING! HTTP mode is completely untested as I cannot make it work on
15
+ # FreeBSD. Please send patches/tests if you find bugs.
16
+ #--
17
+ # TODO dup'd content in MogileFS::NFSFile
18
+
19
+ class MogileFS::HTTPFile < StringIO
20
+
21
+ ##
22
+ # The path this file will be stored to.
23
+
24
+ attr_reader :path
25
+
26
+ ##
27
+ # The key for this file. This key won't represent a real file until you've
28
+ # called #close.
29
+
30
+ attr_reader :key
31
+
32
+ ##
33
+ # The class of this file.
34
+
35
+ attr_reader :class
36
+
37
+ ##
38
+ # Works like File.open. Use MogileFS::MogileFS#new_file instead of this
39
+ # method.
40
+
41
+ def self.open(*args)
42
+ fp = new(*args)
43
+
44
+ return fp unless block_given?
45
+
46
+ begin
47
+ yield fp
48
+ ensure
49
+ fp.close
50
+ end
51
+ end
52
+
53
+ ##
54
+ # Creates a new HTTPFile with MogileFS-specific data. Use
55
+ # MogileFS::MogileFS#new_file instead of this method.
56
+
57
+ def initialize(mg, fid, path, devid, klass, key, dests, content_length)
58
+ super ''
59
+ @mg = mg
60
+ @fid = fid
61
+ @path = path
62
+ @devid = devid
63
+ @klass = klass
64
+ @key = key
65
+
66
+ @dests = dests.map { |(_,u)| URI.parse u }
67
+ @tried = {}
68
+
69
+ @socket = nil
70
+ end
71
+
72
+ ##
73
+ # Closes the file handle and marks it as closed in MogileFS.
74
+
75
+ def close
76
+ connect_socket
77
+
78
+ @socket.write "PUT #{@path.request_uri} HTTP/1.0\r\nContent-length: #{length}\r\n\r\n#{string}"
79
+
80
+ if connected? then
81
+ line = @socket.gets
82
+ raise 'Unable to read response line from server' if line.nil?
83
+
84
+ if line =~ %r%^HTTP/\d+\.\d+\s+(\d+)% then
85
+ status = Integer $1
86
+ case status
87
+ when 200..299 then # success!
88
+ else
89
+ found_header = false
90
+ body = []
91
+ line = @socket.gets
92
+ until line.nil? do
93
+ line.strip
94
+ found_header = true if line.nil?
95
+ next unless found_header
96
+ body << " #{line}"
97
+ end
98
+ body = body[0, 512] if body.length > 512
99
+ raise "HTTP response status from upload: #{body}"
100
+ end
101
+ else
102
+ raise "Response line not understood: #{line}"
103
+ end
104
+ @socket.close
105
+ end
106
+
107
+ @mg.backend.create_close(:fid => @fid, :devid => @devid,
108
+ :domain => @mg.domain, :key => @key,
109
+ :path => @path, :size => length)
110
+ return nil
111
+ end
112
+
113
+ private
114
+
115
+ def connected?
116
+ return !(@socket.nil? or @socket.closed?)
117
+ end
118
+
119
+ def connect_socket
120
+ return @socket if connected?
121
+
122
+ next_path
123
+
124
+ if @path.nil? then
125
+ @tried.clear
126
+ next_path
127
+ raise 'Unable to open socket to storage node' if @path.nil?
128
+ end
129
+
130
+ @socket = TCPSocket.new @path.host, @path.port
131
+ end
132
+
133
+ def next_path
134
+ @path = nil
135
+ @dests.each do |dest|
136
+ unless @tried.include? dest then
137
+ @path = dest
138
+ return
139
+ end
140
+ end
141
+ end
142
+
143
+ end
144
+
@@ -0,0 +1,214 @@
1
+ require 'open-uri'
2
+ require 'timeout'
3
+
4
+ require 'mogilefs/client'
5
+ require 'mogilefs/nfsfile'
6
+
7
+ ##
8
+ # Timeout error class.
9
+
10
+ class MogileFS::Timeout < Timeout::Error; end
11
+
12
+ ##
13
+ # MogileFS File manipulation client.
14
+
15
+ class MogileFS::MogileFS < MogileFS::Client
16
+
17
+ ##
18
+ # The path to the local MogileFS mount point if you are using NFS mode.
19
+
20
+ attr_reader :root
21
+
22
+ ##
23
+ # The domain of keys for this MogileFS client.
24
+
25
+ attr_reader :domain
26
+
27
+ ##
28
+ # Creates a new MogileFS::MogileFS instance. +args+ must include a key
29
+ # :domain specifying the domain of this client. A key :root will be used to
30
+ # specify the root of the NFS file system.
31
+
32
+ def initialize(args = {})
33
+ @domain = args[:domain]
34
+ @root = args[:root]
35
+
36
+ raise ArgumentError, "you must specify a domain" unless @domain
37
+
38
+ super
39
+ end
40
+
41
+ ##
42
+ # Retrieves the contents of +key+.
43
+
44
+ def get_file_data(key)
45
+ get_paths(key).each do |path|
46
+ next unless path
47
+ case path
48
+ when /^http:\/\// then
49
+ begin
50
+ puts "path: #{path}"
51
+ path = URI.parse path
52
+ data = timeout(5, MogileFS::Timeout) { path.read }
53
+ puts "data: #{data}"
54
+ rescue MogileFS::Timeout
55
+ next
56
+ end
57
+ else
58
+ next unless File.exist? path
59
+ return File.read(path)
60
+ end
61
+ end
62
+
63
+ return nil
64
+ end
65
+
66
+ ##
67
+ # Get the paths for +key+.
68
+
69
+ def get_paths(key, noverify = true, zone = nil)
70
+ noverify = noverify ? 1 : 0
71
+ res = @backend.get_paths(:domain => @domain, :key => key,
72
+ :noverify => noverify, :zone => zone)
73
+
74
+ return nil if res.nil? and @backend.lasterr == 'unknown_key'
75
+ paths = (1..res['paths'].to_i).map { |i| res["path#{i}"] }
76
+ return paths if paths.empty?
77
+ return paths if paths.first =~ /^http:\/\//
78
+ return paths.map { |path| File.join @root, path }
79
+ end
80
+
81
+ ##
82
+ # Creates a new file +key+ in +klass+. +bytes+ is currently unused.
83
+ #
84
+ # The +block+ operates like File.open.
85
+
86
+ def new_file(key, klass, bytes = 0, &block) # :yields: file
87
+ raise 'readonly mogilefs' if readonly?
88
+
89
+ res = @backend.create_open(:domain => @domain, :class => klass,
90
+ :key => key, :multi_dest => 1)
91
+
92
+ raise "#{@backend.lasterr}: #{@backend.lasterrstr}" if res.nil? # HACK
93
+
94
+ dests = nil
95
+
96
+ if res.include? 'dev_count' then # HACK HUH?
97
+ dests = (1..res['dev_count'].to_i).map do |i|
98
+ [res["devid_#{i}"], res["path_#{i}"]]
99
+ end
100
+ else
101
+ # 0x0040: d0e4 4f4b 2064 6576 6964 3d31 2666 6964 ..OK.devid=1&fid
102
+ # 0x0050: 3d33 2670 6174 683d 6874 7470 3a2f 2f31 =3&path=http://1
103
+ # 0x0060: 3932 2e31 3638 2e31 2e37 323a 3735 3030 92.168.1.72:7500
104
+ # 0x0070: 2f64 6576 312f 302f 3030 302f 3030 302f /dev1/0/000/000/
105
+ # 0x0080: 3030 3030 3030 3030 3033 2e66 6964 0d0a 0000000003.fid..
106
+
107
+ dests = [[res['devid'], res['path']]]
108
+ end
109
+
110
+ dest = dests.first
111
+ devid, path = dest
112
+
113
+ case path
114
+ when /^http:\/\// then
115
+ MogileFS::HTTPFile.open(self, res['fid'], path, devid, klass, key,
116
+ dests, bytes, &block)
117
+ else
118
+ MogileFS::NFSFile.open(self, res['fid'], path, devid, klass, key, &block)
119
+ end
120
+ end
121
+
122
+ ##
123
+ # Copies the contents of +file+ into +key+ in class +klass+. +file+ can be
124
+ # either a file name or an object that responds to #read.
125
+
126
+ def store_file(key, klass, file)
127
+ raise 'readonly mogilefs' if readonly?
128
+
129
+ new_file key, klass do |mfp|
130
+ if file.respond_to? :read then
131
+ return copy(file, mfp)
132
+ else
133
+ return File.open(file) { |fp| copy(fp, mfp) }
134
+ end
135
+ end
136
+ end
137
+
138
+ ##
139
+ # Stores +content+ into +key+ in class +klass+.
140
+
141
+ def store_content(key, klass, content)
142
+ raise 'readonly mogilefs' if readonly?
143
+
144
+ new_file key, klass do |mfp|
145
+ mfp << content
146
+ end
147
+
148
+ return content.length
149
+ end
150
+
151
+ ##
152
+ # Removes +key+.
153
+
154
+ def delete(key)
155
+ raise 'readonly mogilefs' if readonly?
156
+
157
+ res = @backend.delete :domain => @domain, :key => key
158
+
159
+ if res.nil? and @backend.lasterr != 'unknown_key' then
160
+ raise "unable to delete #{key}: #{@backend.lasterr}"
161
+ end
162
+ end
163
+
164
+ ##
165
+ # Sleeps +duration+.
166
+
167
+ def sleep(duration)
168
+ @backend.sleep :duration => duration
169
+ end
170
+
171
+ ##
172
+ # Renames a key +from+ to key +to+.
173
+
174
+ def rename(from, to)
175
+ raise 'readonly mogilefs' if readonly?
176
+
177
+ res = @backend.rename :domain => @domain, :from_key => from, :to_key => to
178
+
179
+ if res.nil? and @backend.lasterr != 'unknown_key' then
180
+ raise "unable to rename #{from_key} to #{to_key}: #{@backend.lasterr}"
181
+ end
182
+ end
183
+
184
+ ##
185
+ # Lists keys starting with +prefix+ follwing +after+ up to +limit+. If
186
+ # +after+ is nil the list starts at the beginning.
187
+
188
+ def list_keys(prefix, after = nil, limit = 1000)
189
+ res = @backend.list_keys(:domain => domain, :prefix => prefix,
190
+ :after => after, :limit => limit)
191
+
192
+ return nil if res.nil?
193
+
194
+ keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
195
+
196
+ return keys, res['next_after']
197
+ end
198
+
199
+ private
200
+
201
+ def copy(from, to) # HACK use FileUtils
202
+ bytes = 0
203
+
204
+ until from.eof? do
205
+ chunk = from.read 8192
206
+ to.write chunk
207
+ bytes += chunk.length
208
+ end
209
+
210
+ return bytes
211
+ end
212
+
213
+ end
214
+
@@ -0,0 +1,81 @@
1
+ require 'mogilefs/backend'
2
+
3
+ ##
4
+ # NFSFile wraps up the new file operations for storing files onto an NFS
5
+ # storage node.
6
+ #
7
+ # You really don't want to create an NFSFile by hand. Instead you want to
8
+ # create a new file using MogileFS::MogileFS.new_file.
9
+
10
+ class MogileFS::NFSFile < File
11
+
12
+ ##
13
+ # The path of this file not including the local mount point.
14
+
15
+ attr_reader :path
16
+
17
+ ##
18
+ # The key for this file. This key won't represent a real file until you've
19
+ # called #close.
20
+
21
+ attr_reader :key
22
+
23
+ ##
24
+ # The class of this file.
25
+
26
+ attr_reader :class
27
+
28
+ class << self
29
+
30
+ ##
31
+ # Wraps up File.new with MogileFS-specific data. Use
32
+ # MogileFS::MogileFS#new_file instead of this method.
33
+
34
+ def new(mg, fid, path, devid, klass, key)
35
+ fp = super join(mg.root, path), 'w+'
36
+ fp.send :setup, mg, fid, path, devid, klass, key
37
+ return fp
38
+ end
39
+
40
+ ##
41
+ # Wraps up File.open with MogileFS-specific data. Use
42
+ # MogileFS::MogileFS#new_file instead of this method.
43
+
44
+ def open(mg, fid, path, devid, klass, key, &block)
45
+ fp = new mg, fid, path, devid, klass, key
46
+
47
+ return fp if block.nil?
48
+
49
+ begin
50
+ yield fp
51
+ ensure
52
+ fp.close
53
+ end
54
+ end
55
+
56
+ end
57
+
58
+ ##
59
+ # Closes the file handle and marks it as closed in MogileFS.
60
+
61
+ def close
62
+ super
63
+ @mg.backend.create_close(:fid => @fid, :devid => @devid,
64
+ :domain => @mg.domain, :key => @key,
65
+ :path => @path)
66
+ return nil
67
+ end
68
+
69
+ private
70
+
71
+ def setup(mg, fid, path, devid, klass, key)
72
+ @mg = mg
73
+ @fid = fid
74
+ @path = path
75
+ @devid = devid
76
+ @klass = klass
77
+ @key = key
78
+ end
79
+
80
+ end
81
+