mogilefs-client 3.3.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/GIT-VERSION-GEN CHANGED
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
  CONSTANT = "MogileFS::VERSION"
3
3
  RVF = "lib/mogilefs/version.rb"
4
- DEF_VER = "v3.3.0"
4
+ DEF_VER = "v3.4.0"
5
5
  vn = DEF_VER
6
6
 
7
7
  # First see if there is a version file (included in release tarballs),
@@ -38,8 +38,9 @@ class MogileFS::Admin < MogileFS::Client
38
38
 
39
39
  def get_hosts(hostid = nil)
40
40
  to_i = %w(hostid http_port http_get_port)
41
- clean('hosts', 'host',
42
- @backend.get_hosts(hostid ? { :hostid => hostid } : {}), true, to_i)
41
+ want = %w(status hostip hostname altip altmask).concat(to_i)
42
+ rv = @backend.get_hosts(hostid ? { :hostid => hostid } : {})
43
+ clean('hosts', 'host', rv, true, to_i, want)
43
44
  end
44
45
 
45
46
  ##
@@ -56,15 +57,26 @@ class MogileFS::Admin < MogileFS::Client
56
57
  # "devid"=>1,
57
58
  # "hostid"=>1,
58
59
  # "mb_used"=>666,
60
+ # "utilization"=>0.0,
61
+ # "reject_bad_md5"=>false,
62
+ # "observed_state"=>"writeable",
59
63
  # "mb_total"=>666666}]
60
64
 
61
65
  def get_devices(devid = nil)
62
66
  to_i = %w(mb_asof mb_free mb_used mb_total devid weight hostid)
67
+ want = %w(status observed_state).concat(to_i)
63
68
  rv = @backend.get_devices(devid ? { :devid => devid } : {})
64
- rv = clean('devices', 'dev', rv, true, to_i)
69
+ rv = clean('devices', 'dev', rv, true, to_i, want)
70
+ ostates = Hash[%w(readable writeable unreachable).map! { |f| [f,f] }]
71
+
65
72
  rv.each do |row|
66
73
  u = row["utilization"] and
67
74
  row["utilization"] = nil == u ? nil : u.to_f
75
+
76
+ # maps "" to nil (for dead devices)
77
+ row["observed_state"] = ostates[row["observed_state"]]
78
+
79
+ # be sure we do not set this at all for pre-2.60 MogileFS-Server
68
80
  case row["reject_bad_md5"]
69
81
  when "1"
70
82
  row["reject_bad_md5"] = true
@@ -96,9 +108,10 @@ class MogileFS::Admin < MogileFS::Client
96
108
 
97
109
  def list_fids(from_fid, count = 100)
98
110
  to_i = %w(fid devcount length)
111
+ want = %w(domain class key).concat(to_i)
112
+ rv = @backend.list_fids(:from => from_fid, :to => count)
99
113
  # :to is now :count internally in mogilefsd
100
- clean('fid_count', 'fid_',
101
- @backend.list_fids(:from => from_fid, :to => count), true, to_i)
114
+ clean('fid_count', 'fid_', rv, true, to_i, want)
102
115
  end
103
116
 
104
117
  ##
@@ -151,7 +164,8 @@ class MogileFS::Admin < MogileFS::Client
151
164
  # "test" => {
152
165
  # "default" => {
153
166
  # "mindevcount" => 2,
154
- # "replpolicy" => "MultipleHosts()"
167
+ # "replpolicy" => "MultipleHosts()",
168
+ # "hashtype => nil,
155
169
  # }
156
170
  # }
157
171
  # }
@@ -166,12 +180,13 @@ class MogileFS::Admin < MogileFS::Client
166
180
 
167
181
  domains = {}
168
182
  to_i = %w(mindevcount)
183
+ want = %w(name replpolicy hashtype mindevcount)
169
184
  (1..res['domains'].to_i).each do |i|
170
- domain = clean "domain#{i}classes", "domain#{i}class", res, false, to_i
171
-
172
- tmp = domains[res["domain#{i}"].freeze] = {}
185
+ domain = clean("domain#{i}classes", "domain#{i}class", res, false, to_i,
186
+ want)
187
+ tmp = domains[res["domain#{i}"]] = {}
173
188
  domain.each do |d|
174
- tmp[d.delete("name").freeze] = d
189
+ tmp[d.delete("name")] = d
175
190
  have_replpolicy ||= d.include?("replpolicy")
176
191
  end
177
192
  end
@@ -370,23 +385,33 @@ class MogileFS::Admin < MogileFS::Client
370
385
  # "altip"=>"",
371
386
  # "altmask"=>""}]
372
387
 
373
- def clean(count, prefix, res, underscore = true, to_i = [])
388
+ def clean(count, prefix, res, underscore = true, to_i = [], want = nil)
374
389
  empty = ""
375
390
  underscore = underscore ? '_' : empty
376
391
 
377
392
  # convert array to hash for O(1) lookups
378
393
  to_i = to_i.inject({}) { |m,k| m[k] = m }
379
-
380
- keys = res.keys
381
- (1..res[count].to_i).map do |i|
382
- re = /^#{prefix}#{i}#{underscore}/
383
- row = {}
384
- keys.grep(re).each do |k|
385
- v = res[k]
386
- k = k.sub(re, empty).freeze
387
- row[k] = to_i.include?(k) ? (empty == v ? nil : v.to_i) : v
394
+ if want
395
+ (1..res[count].to_i).map do |i|
396
+ row = {}
397
+ want.each do |k|
398
+ v = res["#{prefix}#{i}#{underscore}#{k}"] or next
399
+ row[k] = to_i.include?(k) ? (empty == v ? nil : v.to_i) : v
400
+ end
401
+ row
402
+ end
403
+ else
404
+ keys = res.keys
405
+ (1..res[count].to_i).map do |i|
406
+ re = /^#{prefix}#{i}#{underscore}/
407
+ row = {}
408
+ keys.grep(re).each do |k|
409
+ v = res[k]
410
+ k = k.sub(re, empty)
411
+ row[k] = to_i.include?(k) ? (empty == v ? nil : v.to_i) : v
412
+ end
413
+ row
388
414
  end
389
- row
390
415
  end
391
416
  end
392
417
 
@@ -88,6 +88,57 @@ class MogileFS::MogileFS < MogileFS::Client
88
88
  nil
89
89
  end
90
90
 
91
+ # Enumerates keys and yields a +file_info+ hash for each key matched by
92
+ # +prefix+
93
+ def each_file_info(prefix = "", args = nil)
94
+ # FIXME: there's a lot of duplicate code from list_keys_verbose here...
95
+ raise ArgumentError, "need block" unless block_given?
96
+ ordered = ready = nil
97
+ on_file_info = lambda do |info|
98
+ Hash === info or raise info
99
+ file_info_cleanup(info)
100
+
101
+ # deal with trackers with multiple queryworkers responding out-of-order
102
+ ready[info["key"]] = info
103
+ while info = ready.delete(ordered[-1])
104
+ ordered.pop
105
+ yield info
106
+ end
107
+ end
108
+
109
+ nr = 0
110
+ opts = { :domain => @domain }
111
+ opts[:devices] = 1 if args && args[:devices]
112
+ after = args ? args[:after] : nil
113
+ limit = args ? args[:limit] : nil
114
+
115
+ begin
116
+ keys, after = list_keys(prefix, after, limit || 1000)
117
+ return nr unless keys && keys[0]
118
+ ordered = keys.reverse
119
+ ready = {}
120
+ nr += keys.size
121
+ limit -= keys.size if limit
122
+
123
+ keys.each do |key|
124
+ opts[:key] = key
125
+ @backend.pipeline_dispatch(:file_info, opts, &on_file_info)
126
+ end
127
+ @backend.pipeline_wait
128
+ rescue MogileFS::PipelineError, SystemCallError,
129
+ MogileFS::RequestTruncatedError,
130
+ MogileFS::UnreadableSocketError,
131
+ MogileFS::InvalidResponseError, # truncated response
132
+ MogileFS::Timeout
133
+ @backend.shutdown
134
+ keys = (ordered - ready.keys).reverse!
135
+ retry
136
+ end while limit == nil || limit > 0
137
+ rescue
138
+ @backend.shutdown
139
+ raise
140
+ end
141
+
91
142
  # Retrieves the contents of +key+. If +dst+ is specified, +dst+
92
143
  # should be an IO-like object capable of receiving the +write+ method
93
144
  # or a path name. +copy_length+ may be specified to limit the number of
@@ -339,7 +390,7 @@ class MogileFS::MogileFS < MogileFS::Client
339
390
 
340
391
  def list_keys_verbose(keys, block) # :nodoc:
341
392
  # emulate the MogileFS::Mysql interface, slowly...
342
- ordered = keys.dup
393
+ ordered = keys.reverse
343
394
  ready = {}
344
395
  on_file_info = lambda do |info|
345
396
  Hash === info or raise info
@@ -347,8 +398,8 @@ class MogileFS::MogileFS < MogileFS::Client
347
398
 
348
399
  # deal with trackers with multiple queryworkers responding out-of-order
349
400
  ready[info["key"]] = info
350
- while info = ready.delete(ordered[0])
351
- block.call(ordered.shift, info["length"], info["devcount"])
401
+ while info = ready.delete(ordered[-1])
402
+ block.call(ordered.pop, info["length"], info["devcount"])
352
403
  end
353
404
  end
354
405
  opts = { :domain => @domain }
@@ -371,7 +422,7 @@ class MogileFS::MogileFS < MogileFS::Client
371
422
  MogileFS::InvalidResponseError, # truncated response
372
423
  MogileFS::Timeout
373
424
  @backend.shutdown
374
- keys = ordered - ready.keys
425
+ keys = (ordered - ready.keys).reverse!
375
426
  retry
376
427
  rescue
377
428
  @backend.shutdown
@@ -36,5 +36,21 @@ class TestMogileFSIntegrationListKeys < TestMogIntegration
36
36
  assert_equal "ek_#{n.to_s}", key
37
37
  n += 1
38
38
  end
39
+ assert_equal 9, n
40
+ end
41
+
42
+ def test_each_file_info
43
+ 9.times { |i| @client.store_content("ek_#{i}", nil, i.to_s) }
44
+ n = 0
45
+ @client.each_file_info do |info|
46
+ assert_equal @client.domain, info["domain"]
47
+ assert_equal n.to_s.size, info["length"]
48
+ assert_kind_of Integer, info["fid"]
49
+ assert_kind_of Integer, info["devcount"]
50
+ assert_equal "default", info["class"]
51
+ assert_equal "ek_#{n}", info["key"]
52
+ n += 1
53
+ end
54
+ assert_equal 9, n
39
55
  end
40
56
  end
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mogilefs-client
3
3
  version: !ruby/object:Gem::Version
4
- hash: 11
4
+ hash: 23
5
5
  prerelease:
6
6
  segments:
7
7
  - 3
8
- - 3
8
+ - 4
9
9
  - 0
10
- version: 3.3.0
10
+ version: 3.4.0
11
11
  platform: ruby
12
12
  authors:
13
13
  - Eric Wong
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2012-08-11 00:00:00 Z
18
+ date: 2012-10-09 00:00:00 Z
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency
21
21
  name: rdoc