pager-mogilefs-client 1.2.1.20080519

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,81 @@
1
+ require 'mogilefs/backend'
2
+
3
+ ##
4
+ # NFSFile wraps up the new file operations for storing files onto an NFS
5
+ # storage node.
6
+ #
7
+ # You really don't want to create an NFSFile by hand. Instead you want to
8
+ # create a new file using MogileFS::MogileFS.new_file.
9
+
10
+ class MogileFS::NFSFile < File
11
+
12
+ ##
13
+ # The path of this file not including the local mount point.
14
+
15
+ attr_reader :path
16
+
17
+ ##
18
+ # The key for this file. This key won't represent a real file until you've
19
+ # called #close.
20
+
21
+ attr_reader :key
22
+
23
+ ##
24
+ # The class of this file.
25
+
26
+ attr_reader :class
27
+
28
+ class << self
29
+
30
+ ##
31
+ # Wraps up File.new with MogileFS-specific data. Use
32
+ # MogileFS::MogileFS#new_file instead of this method.
33
+
34
+ def new(mg, fid, path, devid, klass, key)
35
+ fp = super join(mg.root, path), 'w+'
36
+ fp.send :setup, mg, fid, path, devid, klass, key
37
+ return fp
38
+ end
39
+
40
+ ##
41
+ # Wraps up File.open with MogileFS-specific data. Use
42
+ # MogileFS::MogileFS#new_file instead of this method.
43
+
44
+ def open(mg, fid, path, devid, klass, key, &block)
45
+ fp = new mg, fid, path, devid, klass, key
46
+
47
+ return fp if block.nil?
48
+
49
+ begin
50
+ yield fp
51
+ ensure
52
+ fp.close
53
+ end
54
+ end
55
+
56
+ end
57
+
58
+ ##
59
+ # Closes the file handle and marks it as closed in MogileFS.
60
+
61
+ def close
62
+ super
63
+ @mg.backend.create_close(:fid => @fid, :devid => @devid,
64
+ :domain => @mg.domain, :key => @key,
65
+ :path => @path)
66
+ return nil
67
+ end
68
+
69
+ private
70
+
71
+ def setup(mg, fid, path, devid, klass, key)
72
+ @mg = mg
73
+ @fid = fid
74
+ @path = path
75
+ @devid = devid
76
+ @klass = klass
77
+ @key = key
78
+ end
79
+
80
+ end
81
+
@@ -0,0 +1,50 @@
1
+ require 'thread'
2
+ require 'mogilefs'
3
+
4
+ class MogileFS::Pool
5
+
6
+ class BadObjectError < RuntimeError; end
7
+
8
+ def initialize(klass, *args)
9
+ @args = args
10
+ @klass = klass
11
+ @queue = Queue.new
12
+ @objects = []
13
+ end
14
+
15
+ def get
16
+ begin
17
+ object = @queue.pop true
18
+ rescue ThreadError
19
+ object = @klass.new(*@args)
20
+ @objects << object
21
+ end
22
+ return object
23
+ end
24
+
25
+ def put(o)
26
+ raise BadObjectError unless @objects.include? o
27
+ @queue.push o
28
+ purge
29
+ end
30
+
31
+ def use
32
+ object = get
33
+ yield object
34
+ ensure
35
+ put object
36
+ end
37
+
38
+ def purge
39
+ return if @queue.length < 5
40
+ begin
41
+ until @queue.length <= 2 do
42
+ obj = @queue.pop true
43
+ @objects.delete obj
44
+ end
45
+ rescue ThreadError
46
+ end
47
+ end
48
+
49
+ end
50
+
@@ -0,0 +1,54 @@
1
+ require 'test/unit'
2
+
3
+ $TESTING = true
4
+
5
+ require 'mogilefs'
6
+
7
+ class FakeBackend
8
+
9
+ attr_reader :lasterr, :lasterrstr
10
+
11
+ def initialize
12
+ @responses = Hash.new { |h,k| h[k] = [] }
13
+ @lasterr = nil
14
+ @lasterrstr = nil
15
+ end
16
+
17
+ def method_missing(meth, *args)
18
+ meth = meth.to_s
19
+ if meth =~ /(.*)=$/ then
20
+ @responses[$1] << args.first
21
+ else
22
+ response = @responses[meth].shift
23
+ case response
24
+ when Array then
25
+ @lasterr = response.first
26
+ @lasterrstr = response.last
27
+ return nil
28
+ end
29
+ return response
30
+ end
31
+ end
32
+
33
+ end
34
+
35
+ class MogileFS::Client
36
+ attr_writer :readonly
37
+ end
38
+
39
+ class TestMogileFS < Test::Unit::TestCase
40
+
41
+ def setup
42
+ return if self.class == TestMogileFS
43
+ @root = '/mogilefs/test'
44
+ @client = @klass.new :hosts => ['kaa:6001'], :domain => 'test',
45
+ :root => @root
46
+ @backend = FakeBackend.new
47
+ @client.instance_variable_set '@backend', @backend
48
+ end
49
+
50
+ def test_nothing
51
+ end
52
+
53
+ end
54
+
@@ -0,0 +1,174 @@
1
+ require 'test/setup'
2
+
3
+ class TestMogileFS__Admin < TestMogileFS
4
+
5
+ def setup
6
+ @klass = MogileFS::Admin
7
+ super
8
+ end
9
+
10
+ def test_clean
11
+ res = {"host1_remoteroot"=>"/mnt/mogilefs/rur-1",
12
+ "host1_hostname"=>"rur-1",
13
+ "host1_hostid"=>"1",
14
+ "host1_http_get_port"=>"",
15
+ "host1_altip"=>"",
16
+ "hosts"=>"1",
17
+ "host1_hostip"=>"",
18
+ "host1_http_port"=>"",
19
+ "host1_status"=>"alive",
20
+ "host1_altmask"=>""}
21
+ actual = @client.clean 'hosts', 'host', res
22
+
23
+ expected = [{"status"=>"alive",
24
+ "http_get_port"=>"",
25
+ "http_port"=>"",
26
+ "hostid"=>"1",
27
+ "hostip"=>"",
28
+ "hostname"=>"rur-1",
29
+ "remoteroot"=>"/mnt/mogilefs/rur-1",
30
+ "altip"=>"",
31
+ "altmask"=>""}]
32
+
33
+ assert_equal expected, actual
34
+ end
35
+
36
+ def test_each_fid
37
+ @backend.stats = {
38
+ 'fidmax' => '182',
39
+ 'fidcount' => '2',
40
+ }
41
+
42
+ @backend.list_fids = {
43
+ 'fid_count' => '1',
44
+ 'fid_1_fid' => '99',
45
+ 'fid_1_class' => 'normal',
46
+ 'fid_1_devcount' => '2',
47
+ 'fid_1_domain' => 'test',
48
+ 'fid_1_key' => 'file_key',
49
+ 'fid_1_length' => '4',
50
+ }
51
+
52
+ @backend.list_fids = {
53
+ 'fid_count' => '1',
54
+ 'fid_1_fid' => '182',
55
+ 'fid_1_class' => 'normal',
56
+ 'fid_1_devcount' => '2',
57
+ 'fid_1_domain' => 'test',
58
+ 'fid_1_key' => 'new_new_key',
59
+ 'fid_1_length' => '9',
60
+ }
61
+
62
+ fids = []
63
+ @client.each_fid { |fid| fids << fid }
64
+
65
+ expected = [
66
+ { "fid" => "99",
67
+ "class" => "normal",
68
+ "domain" => "test",
69
+ "devcount" => "2",
70
+ "length" => "4",
71
+ "key" => "file_key" },
72
+ { "fid" => "182",
73
+ "class" => "normal",
74
+ "devcount" => "2",
75
+ "domain" => "test",
76
+ "length" => "9",
77
+ "key" => "new_new_key" },
78
+ ]
79
+
80
+ assert_equal expected, fids
81
+ end
82
+
83
+ def test_get_domains
84
+ @backend.get_domains = {
85
+ 'domains' => 2,
86
+ 'domain1' => 'test',
87
+ 'domain2' => 'images',
88
+ 'domain1classes' => '1',
89
+ 'domain2classes' => '2',
90
+ 'domain1class1name' => 'default',
91
+ 'domain1class1mindevcount' => '2',
92
+ 'domain2class1name' => 'default',
93
+ 'domain2class1mindevcount' => '2',
94
+ 'domain2class2name' => 'resize',
95
+ 'domain2class2mindevcount' => '1',
96
+ }
97
+
98
+ expected = {
99
+ 'test' => { 'default' => 2, },
100
+ 'images' => { 'default' => 2, 'resize' => 1 },
101
+ }
102
+
103
+ assert_equal expected, @client.get_domains
104
+ end
105
+
106
+ def disabled_test_get_stats
107
+ @backend.stats = {}
108
+
109
+ expected = {
110
+ 'fids' => { 'max' => '99', 'count' => '2' },
111
+ 'device' => [
112
+ { 'status' => 'alive', 'files' => '2', 'id' => '1', 'host' => 'rur-1' },
113
+ { 'status' => 'alive', 'files' => '2', 'id' => '2', 'host' => 'rur-2' }
114
+ ],
115
+ 'replication' => [
116
+ { 'files' => '2', 'class' => 'normal', 'devcount' => '2',
117
+ 'domain' => 'test' }
118
+ ],
119
+ 'file' => [{ 'files' => '2', 'class' => 'normal', 'domain' => 'test' }]
120
+ }
121
+
122
+ assert_equal
123
+ end
124
+
125
+ def test_get_stats_fids
126
+ @backend.stats = {
127
+ 'fidmax' => 99,
128
+ 'fidcount' => 2,
129
+ }
130
+
131
+ expected = {
132
+ 'fids' => { 'max' => 99, 'count' => 2 },
133
+ }
134
+
135
+ assert_equal expected, @client.get_stats('all')
136
+ end
137
+
138
+ def test_list_fids
139
+ @backend.list_fids = {
140
+ 'fid_count' => '2',
141
+ 'fid_1_fid' => '99',
142
+ 'fid_1_class' => 'normal',
143
+ 'fid_1_devcount' => '2',
144
+ 'fid_1_domain' => 'test',
145
+ 'fid_1_key' => 'file_key',
146
+ 'fid_1_length' => '4',
147
+ 'fid_2_fid' => '82',
148
+ 'fid_2_class' => 'normal',
149
+ 'fid_2_devcount' => '2',
150
+ 'fid_2_domain' => 'test',
151
+ 'fid_2_key' => 'new_new_key',
152
+ 'fid_2_length' => '9',
153
+ }
154
+
155
+ expected = [
156
+ { "fid" => "99",
157
+ "class" => "normal",
158
+ "domain" => "test",
159
+ "devcount" => "2",
160
+ "length" => "4",
161
+ "key" => "file_key" },
162
+ { "fid" => "82",
163
+ "class" => "normal",
164
+ "devcount" => "2",
165
+ "domain" => "test",
166
+ "length" => "9",
167
+ "key" => "new_new_key" },
168
+ ]
169
+
170
+ assert_equal expected, @client.list_fids(0, 100)
171
+ end
172
+
173
+ end
174
+
@@ -0,0 +1,220 @@
1
+ require 'test/unit'
2
+
3
+ $TESTING = true
4
+
5
+ require 'mogilefs/backend'
6
+
7
+ class MogileFS::Backend
8
+
9
+ attr_accessor :hosts
10
+ attr_reader :timeout, :dead
11
+ attr_writer :lasterr, :lasterrstr, :socket
12
+
13
+ end
14
+
15
+ class FakeSocket
16
+
17
+ def initialize
18
+ @closed = false
19
+ end
20
+
21
+ def closed?
22
+ @closed
23
+ end
24
+
25
+ def close
26
+ @closed = true
27
+ return nil
28
+ end
29
+
30
+ def peeraddr
31
+ ['AF_INET', 6001, 'localhost', '127.0.0.1']
32
+ end
33
+
34
+ end
35
+
36
+ class TestBackend < Test::Unit::TestCase
37
+
38
+ def setup
39
+ @backend = MogileFS::Backend.new :hosts => ['localhost:1']
40
+ end
41
+
42
+ def test_initialize
43
+ assert_raises ArgumentError do MogileFS::Backend.new end
44
+ assert_raises ArgumentError do MogileFS::Backend.new :hosts => [] end
45
+ assert_raises ArgumentError do MogileFS::Backend.new :hosts => [''] end
46
+
47
+ assert_equal ['localhost:1'], @backend.hosts
48
+ assert_equal 3, @backend.timeout
49
+ assert_equal nil, @backend.lasterr
50
+ assert_equal nil, @backend.lasterrstr
51
+ assert_equal({}, @backend.dead)
52
+
53
+ @backend = MogileFS::Backend.new :hosts => ['localhost:6001'], :timeout => 1
54
+ assert_equal 1, @backend.timeout
55
+ end
56
+
57
+ def test_do_request
58
+ socket_request = ''
59
+ socket = Object.new
60
+ def socket.closed?() false end
61
+ def socket.send(request, flags) return request.length end
62
+ def @backend.select(*args) return [true] end
63
+ def socket.gets() return 'OK 1 you=win' end
64
+
65
+ @backend.instance_variable_set '@socket', socket
66
+
67
+ assert_equal({'you' => 'win'},
68
+ @backend.do_request('go!', { 'fight' => 'team fight!' }))
69
+ end
70
+
71
+ def test_do_request_send_error
72
+ socket_request = ''
73
+ socket = Object.new
74
+ def socket.closed?() false end
75
+ def socket.send(request, flags) raise SystemCallError, 'dummy' end
76
+
77
+ @backend.instance_variable_set '@socket', socket
78
+
79
+ assert_raises RuntimeError do
80
+ @backend.do_request 'go!', { 'fight' => 'team fight!' }
81
+ end
82
+
83
+ assert_equal nil, @backend.instance_variable_get('@socket')
84
+ end
85
+
86
+ def test_do_request_truncated
87
+ socket_request = ''
88
+ socket = Object.new
89
+ def socket.closed?() false end
90
+ def socket.send(request, flags) return request.length - 1 end
91
+
92
+ @backend.instance_variable_set '@socket', socket
93
+
94
+ assert_raises RuntimeError do
95
+ @backend.do_request 'go!', { 'fight' => 'team fight!' }
96
+ end
97
+ end
98
+
99
+ def test_make_request
100
+ assert_equal "go! fight=team+fight%21\r\n",
101
+ @backend.make_request('go!', { 'fight' => 'team fight!' })
102
+ end
103
+
104
+ def test_parse_response
105
+ assert_equal({'foo' => 'bar', 'baz' => 'hoge'},
106
+ @backend.parse_response('OK 1 foo=bar&baz=hoge'))
107
+ assert_equal nil, @backend.parse_response('ERR you totally suck')
108
+ assert_equal 'you', @backend.lasterr
109
+ assert_equal 'totally suck', @backend.lasterrstr
110
+
111
+ assert_raises RuntimeError do
112
+ @backend.parse_response 'garbage'
113
+ end
114
+ end
115
+
116
+ def test_readable_eh_readable
117
+ socket = Object.new
118
+ def socket.closed?() false end
119
+ def @backend.select(*args) return [true] end
120
+ @backend.instance_variable_set '@socket', socket
121
+
122
+ assert_equal true, @backend.readable?
123
+ end
124
+
125
+ def test_readable_eh_not_readable
126
+ socket = FakeSocket.new
127
+ def socket.closed?() false end
128
+ def @backend.select(*args) return [] end
129
+ @backend.instance_variable_set '@socket', socket
130
+
131
+ begin
132
+ @backend.readable?
133
+ rescue MogileFS::UnreadableSocketError => e
134
+ assert_equal '127.0.0.1:6001 never became readable', e.message
135
+ rescue Exception
136
+ flunk "MogileFS::UnreadableSocketError not raised"
137
+ else
138
+ flunk "MogileFS::UnreadableSocketError not raised"
139
+ end
140
+ end
141
+
142
+ def test_socket
143
+ assert_equal({}, @backend.dead)
144
+ assert_raises RuntimeError do @backend.socket end
145
+ assert_equal(['localhost:1'], @backend.dead.keys)
146
+ end
147
+
148
+ def test_socket_robust
149
+ @backend.hosts = ['localhost:6001', 'localhost:6002']
150
+ def @backend.connect_to(host, port)
151
+ @first = (defined? @first) ? false : true
152
+ raise Errno::ECONNREFUSED if @first
153
+ end
154
+
155
+ assert_equal({}, @backend.dead)
156
+ @backend.socket
157
+ assert_equal false, @backend.dead.keys.empty?
158
+ end
159
+
160
+ def test_shutdown
161
+ fake_socket = FakeSocket.new
162
+ @backend.socket = fake_socket
163
+ assert_equal fake_socket, @backend.socket
164
+ @backend.shutdown
165
+ assert_equal nil, @backend.instance_variable_get(:@socket)
166
+ end
167
+
168
+ def test_url_decode
169
+ assert_equal({"\272z" => "\360opy", "f\000" => "\272r"},
170
+ @backend.url_decode("%baz=%f0opy&f%00=%bar"))
171
+ end
172
+
173
+ def test_url_encode
174
+ params = [["f\000", "\272r"], ["\272z", "\360opy"]]
175
+ assert_equal "f%00=%bar&%baz=%f0opy", @backend.url_encode(params)
176
+ end
177
+
178
+ def test_url_escape # \n for unit_diff
179
+ actual = (0..255).map { |c| @backend.url_escape c.chr }.join "\n"
180
+
181
+ expected = []
182
+ expected.push(*(0..0x1f).map { |c| "%%%0.2x" % c })
183
+ expected << '+'
184
+ expected.push(*(0x21..0x2b).map { |c| "%%%0.2x" % c })
185
+ expected.push(*%w[, - . /])
186
+ expected.push(*('0'..'9'))
187
+ expected.push(*%w[: %3b %3c %3d %3e %3f %40])
188
+ expected.push(*('A'..'Z'))
189
+ expected.push(*%w[%5b \\ %5d %5e _ %60])
190
+ expected.push(*('a'..'z'))
191
+ expected.push(*(0x7b..0xff).map { |c| "%%%0.2x" % c })
192
+
193
+ expected = expected.join "\n"
194
+
195
+ assert_equal expected, actual
196
+ end
197
+
198
+ def test_url_unescape
199
+ input = []
200
+ input.push(*(0..0x1f).map { |c| "%%%0.2x" % c })
201
+ input << '+'
202
+ input.push(*(0x21..0x2b).map { |c| "%%%0.2x" % c })
203
+ input.push(*%w[, - . /])
204
+ input.push(*('0'..'9'))
205
+ input.push(*%w[: %3b %3c %3d %3e %3f %40])
206
+ input.push(*('A'..'Z'))
207
+ input.push(*%w[%5b \\ %5d %5e _ %60])
208
+ input.push(*('a'..'z'))
209
+ input.push(*(0x7b..0xff).map { |c| "%%%0.2x" % c })
210
+
211
+ actual = input.map { |c| @backend.url_unescape c }.join "\n"
212
+
213
+ expected = (0..255).map { |c| c.chr }.join "\n"
214
+ expected.sub! '+', ' '
215
+
216
+ assert_equal expected, actual
217
+ end
218
+
219
+ end
220
+