mogilefs-client 3.3.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/GIT-VERSION-GEN +1 -1
- data/lib/mogilefs/admin.rb +46 -21
- data/lib/mogilefs/mogilefs.rb +55 -4
- data/test/test_mogilefs_integration_list_keys.rb +16 -0
- metadata +4 -4
data/GIT-VERSION-GEN
CHANGED
data/lib/mogilefs/admin.rb
CHANGED
@@ -38,8 +38,9 @@ class MogileFS::Admin < MogileFS::Client
|
|
38
38
|
|
39
39
|
def get_hosts(hostid = nil)
|
40
40
|
to_i = %w(hostid http_port http_get_port)
|
41
|
-
|
42
|
-
|
41
|
+
want = %w(status hostip hostname altip altmask).concat(to_i)
|
42
|
+
rv = @backend.get_hosts(hostid ? { :hostid => hostid } : {})
|
43
|
+
clean('hosts', 'host', rv, true, to_i, want)
|
43
44
|
end
|
44
45
|
|
45
46
|
##
|
@@ -56,15 +57,26 @@ class MogileFS::Admin < MogileFS::Client
|
|
56
57
|
# "devid"=>1,
|
57
58
|
# "hostid"=>1,
|
58
59
|
# "mb_used"=>666,
|
60
|
+
# "utilization"=>0.0,
|
61
|
+
# "reject_bad_md5"=>false,
|
62
|
+
# "observed_state"=>"writeable",
|
59
63
|
# "mb_total"=>666666}]
|
60
64
|
|
61
65
|
def get_devices(devid = nil)
|
62
66
|
to_i = %w(mb_asof mb_free mb_used mb_total devid weight hostid)
|
67
|
+
want = %w(status observed_state).concat(to_i)
|
63
68
|
rv = @backend.get_devices(devid ? { :devid => devid } : {})
|
64
|
-
rv = clean('devices', 'dev', rv, true, to_i)
|
69
|
+
rv = clean('devices', 'dev', rv, true, to_i, want)
|
70
|
+
ostates = Hash[%w(readable writeable unreachable).map! { |f| [f,f] }]
|
71
|
+
|
65
72
|
rv.each do |row|
|
66
73
|
u = row["utilization"] and
|
67
74
|
row["utilization"] = nil == u ? nil : u.to_f
|
75
|
+
|
76
|
+
# maps "" to nil (for dead devices)
|
77
|
+
row["observed_state"] = ostates[row["observed_state"]]
|
78
|
+
|
79
|
+
# be sure we do not set this at all for pre-2.60 MogileFS-Server
|
68
80
|
case row["reject_bad_md5"]
|
69
81
|
when "1"
|
70
82
|
row["reject_bad_md5"] = true
|
@@ -96,9 +108,10 @@ class MogileFS::Admin < MogileFS::Client
|
|
96
108
|
|
97
109
|
def list_fids(from_fid, count = 100)
|
98
110
|
to_i = %w(fid devcount length)
|
111
|
+
want = %w(domain class key).concat(to_i)
|
112
|
+
rv = @backend.list_fids(:from => from_fid, :to => count)
|
99
113
|
# :to is now :count internally in mogilefsd
|
100
|
-
clean('fid_count', 'fid_',
|
101
|
-
@backend.list_fids(:from => from_fid, :to => count), true, to_i)
|
114
|
+
clean('fid_count', 'fid_', rv, true, to_i, want)
|
102
115
|
end
|
103
116
|
|
104
117
|
##
|
@@ -151,7 +164,8 @@ class MogileFS::Admin < MogileFS::Client
|
|
151
164
|
# "test" => {
|
152
165
|
# "default" => {
|
153
166
|
# "mindevcount" => 2,
|
154
|
-
# "replpolicy" => "MultipleHosts()"
|
167
|
+
# "replpolicy" => "MultipleHosts()",
|
168
|
+
# "hashtype => nil,
|
155
169
|
# }
|
156
170
|
# }
|
157
171
|
# }
|
@@ -166,12 +180,13 @@ class MogileFS::Admin < MogileFS::Client
|
|
166
180
|
|
167
181
|
domains = {}
|
168
182
|
to_i = %w(mindevcount)
|
183
|
+
want = %w(name replpolicy hashtype mindevcount)
|
169
184
|
(1..res['domains'].to_i).each do |i|
|
170
|
-
domain = clean
|
171
|
-
|
172
|
-
tmp = domains[res["domain#{i}"]
|
185
|
+
domain = clean("domain#{i}classes", "domain#{i}class", res, false, to_i,
|
186
|
+
want)
|
187
|
+
tmp = domains[res["domain#{i}"]] = {}
|
173
188
|
domain.each do |d|
|
174
|
-
tmp[d.delete("name")
|
189
|
+
tmp[d.delete("name")] = d
|
175
190
|
have_replpolicy ||= d.include?("replpolicy")
|
176
191
|
end
|
177
192
|
end
|
@@ -370,23 +385,33 @@ class MogileFS::Admin < MogileFS::Client
|
|
370
385
|
# "altip"=>"",
|
371
386
|
# "altmask"=>""}]
|
372
387
|
|
373
|
-
def clean(count, prefix, res, underscore = true, to_i = [])
|
388
|
+
def clean(count, prefix, res, underscore = true, to_i = [], want = nil)
|
374
389
|
empty = ""
|
375
390
|
underscore = underscore ? '_' : empty
|
376
391
|
|
377
392
|
# convert array to hash for O(1) lookups
|
378
393
|
to_i = to_i.inject({}) { |m,k| m[k] = m }
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
394
|
+
if want
|
395
|
+
(1..res[count].to_i).map do |i|
|
396
|
+
row = {}
|
397
|
+
want.each do |k|
|
398
|
+
v = res["#{prefix}#{i}#{underscore}#{k}"] or next
|
399
|
+
row[k] = to_i.include?(k) ? (empty == v ? nil : v.to_i) : v
|
400
|
+
end
|
401
|
+
row
|
402
|
+
end
|
403
|
+
else
|
404
|
+
keys = res.keys
|
405
|
+
(1..res[count].to_i).map do |i|
|
406
|
+
re = /^#{prefix}#{i}#{underscore}/
|
407
|
+
row = {}
|
408
|
+
keys.grep(re).each do |k|
|
409
|
+
v = res[k]
|
410
|
+
k = k.sub(re, empty)
|
411
|
+
row[k] = to_i.include?(k) ? (empty == v ? nil : v.to_i) : v
|
412
|
+
end
|
413
|
+
row
|
388
414
|
end
|
389
|
-
row
|
390
415
|
end
|
391
416
|
end
|
392
417
|
|
data/lib/mogilefs/mogilefs.rb
CHANGED
@@ -88,6 +88,57 @@ class MogileFS::MogileFS < MogileFS::Client
|
|
88
88
|
nil
|
89
89
|
end
|
90
90
|
|
91
|
+
# Enumerates keys and yields a +file_info+ hash for each key matched by
|
92
|
+
# +prefix+
|
93
|
+
def each_file_info(prefix = "", args = nil)
|
94
|
+
# FIXME: there's a lot of duplicate code from list_keys_verbose here...
|
95
|
+
raise ArgumentError, "need block" unless block_given?
|
96
|
+
ordered = ready = nil
|
97
|
+
on_file_info = lambda do |info|
|
98
|
+
Hash === info or raise info
|
99
|
+
file_info_cleanup(info)
|
100
|
+
|
101
|
+
# deal with trackers with multiple queryworkers responding out-of-order
|
102
|
+
ready[info["key"]] = info
|
103
|
+
while info = ready.delete(ordered[-1])
|
104
|
+
ordered.pop
|
105
|
+
yield info
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
nr = 0
|
110
|
+
opts = { :domain => @domain }
|
111
|
+
opts[:devices] = 1 if args && args[:devices]
|
112
|
+
after = args ? args[:after] : nil
|
113
|
+
limit = args ? args[:limit] : nil
|
114
|
+
|
115
|
+
begin
|
116
|
+
keys, after = list_keys(prefix, after, limit || 1000)
|
117
|
+
return nr unless keys && keys[0]
|
118
|
+
ordered = keys.reverse
|
119
|
+
ready = {}
|
120
|
+
nr += keys.size
|
121
|
+
limit -= keys.size if limit
|
122
|
+
|
123
|
+
keys.each do |key|
|
124
|
+
opts[:key] = key
|
125
|
+
@backend.pipeline_dispatch(:file_info, opts, &on_file_info)
|
126
|
+
end
|
127
|
+
@backend.pipeline_wait
|
128
|
+
rescue MogileFS::PipelineError, SystemCallError,
|
129
|
+
MogileFS::RequestTruncatedError,
|
130
|
+
MogileFS::UnreadableSocketError,
|
131
|
+
MogileFS::InvalidResponseError, # truncated response
|
132
|
+
MogileFS::Timeout
|
133
|
+
@backend.shutdown
|
134
|
+
keys = (ordered - ready.keys).reverse!
|
135
|
+
retry
|
136
|
+
end while limit == nil || limit > 0
|
137
|
+
rescue
|
138
|
+
@backend.shutdown
|
139
|
+
raise
|
140
|
+
end
|
141
|
+
|
91
142
|
# Retrieves the contents of +key+. If +dst+ is specified, +dst+
|
92
143
|
# should be an IO-like object capable of receiving the +write+ method
|
93
144
|
# or a path name. +copy_length+ may be specified to limit the number of
|
@@ -339,7 +390,7 @@ class MogileFS::MogileFS < MogileFS::Client
|
|
339
390
|
|
340
391
|
def list_keys_verbose(keys, block) # :nodoc:
|
341
392
|
# emulate the MogileFS::Mysql interface, slowly...
|
342
|
-
ordered = keys.
|
393
|
+
ordered = keys.reverse
|
343
394
|
ready = {}
|
344
395
|
on_file_info = lambda do |info|
|
345
396
|
Hash === info or raise info
|
@@ -347,8 +398,8 @@ class MogileFS::MogileFS < MogileFS::Client
|
|
347
398
|
|
348
399
|
# deal with trackers with multiple queryworkers responding out-of-order
|
349
400
|
ready[info["key"]] = info
|
350
|
-
while info = ready.delete(ordered[
|
351
|
-
block.call(ordered.
|
401
|
+
while info = ready.delete(ordered[-1])
|
402
|
+
block.call(ordered.pop, info["length"], info["devcount"])
|
352
403
|
end
|
353
404
|
end
|
354
405
|
opts = { :domain => @domain }
|
@@ -371,7 +422,7 @@ class MogileFS::MogileFS < MogileFS::Client
|
|
371
422
|
MogileFS::InvalidResponseError, # truncated response
|
372
423
|
MogileFS::Timeout
|
373
424
|
@backend.shutdown
|
374
|
-
keys = ordered - ready.keys
|
425
|
+
keys = (ordered - ready.keys).reverse!
|
375
426
|
retry
|
376
427
|
rescue
|
377
428
|
@backend.shutdown
|
@@ -36,5 +36,21 @@ class TestMogileFSIntegrationListKeys < TestMogIntegration
|
|
36
36
|
assert_equal "ek_#{n.to_s}", key
|
37
37
|
n += 1
|
38
38
|
end
|
39
|
+
assert_equal 9, n
|
40
|
+
end
|
41
|
+
|
42
|
+
def test_each_file_info
|
43
|
+
9.times { |i| @client.store_content("ek_#{i}", nil, i.to_s) }
|
44
|
+
n = 0
|
45
|
+
@client.each_file_info do |info|
|
46
|
+
assert_equal @client.domain, info["domain"]
|
47
|
+
assert_equal n.to_s.size, info["length"]
|
48
|
+
assert_kind_of Integer, info["fid"]
|
49
|
+
assert_kind_of Integer, info["devcount"]
|
50
|
+
assert_equal "default", info["class"]
|
51
|
+
assert_equal "ek_#{n}", info["key"]
|
52
|
+
n += 1
|
53
|
+
end
|
54
|
+
assert_equal 9, n
|
39
55
|
end
|
40
56
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mogilefs-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 23
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 3
|
8
|
-
-
|
8
|
+
- 4
|
9
9
|
- 0
|
10
|
-
version: 3.
|
10
|
+
version: 3.4.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Eric Wong
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-
|
18
|
+
date: 2012-10-09 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: rdoc
|