mogilefs-client 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.document CHANGED
@@ -1,3 +1,4 @@
1
+ NEWS
1
2
  History
2
3
  ChangeLog
3
4
  LICENSE
@@ -9,3 +10,5 @@ lib/mogilefs/admin.rb
9
10
  lib/mogilefs/backend.rb
10
11
  lib/mogilefs/client.rb
11
12
  lib/mogilefs/mogilefs.rb
13
+ lib/mogilefs/new_file.rb
14
+ lib/mogilefs/new_file/writer.rb
data/.gitignore CHANGED
@@ -6,3 +6,4 @@
6
6
  /NEWS
7
7
  /LATEST
8
8
  /lib/mogilefs/version.rb
9
+ /Manifest.txt
data/GIT-VERSION-GEN CHANGED
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
  CONSTANT = "MogileFS::VERSION"
3
3
  RVF = "lib/mogilefs/version.rb"
4
- DEF_VER = "v3.0.0"
4
+ DEF_VER = "v3.1.0"
5
5
  vn = DEF_VER
6
6
 
7
7
  # First see if there is a version file (included in release tarballs),
data/GNUmakefile CHANGED
@@ -34,15 +34,7 @@ $(T):
34
34
  $(run_test)
35
35
 
36
36
  RUBY_VERSION_FILE = lib/mogilefs/version.rb
37
- # using make instead of rake since Rakefile takes too long to load
38
- manifest: Manifest.txt
39
- Manifest.txt:
40
- git ls-files > $@+
41
- echo $(RUBY_VERSION_FILE) >> $@+
42
- cmp $@+ $@ || mv $@+ $@
43
- $(RM) -f $@+
44
-
45
- package: manifest
37
+ package:
46
38
  git diff --exit-code HEAD^0
47
39
  $(RM) -r pkg/
48
40
  rake fix_perms
@@ -55,7 +47,7 @@ flay: $(libs)
55
47
  flay $(flay_flags) $^
56
48
  flog: $(libs)
57
49
  flog $(flog_flags) $^
58
- .PHONY: $(T) Manifest.txt
50
+ .PHONY: $(T)
59
51
 
60
52
  check-warnings:
61
53
  @(for i in $$(git ls-files '*.rb'| grep -v '^setup\.rb$$'); \
data/History CHANGED
@@ -1,3 +1,6 @@
1
+ This document is for historical releases. See the auto-generated
2
+ NEWS document for the latest releases.
3
+
1
4
  = 2.2.0
2
5
  * internal cleanups (no public API breakage)
3
6
  * refactor backend socket/connection handling for reliability
data/Rakefile CHANGED
@@ -2,10 +2,18 @@ require 'rubygems'
2
2
  require 'hoe'
3
3
  load "./GIT-VERSION-GEN"
4
4
 
5
+ include Rake::DSL if defined?(Rake::DSL)
5
6
  $:.unshift 'lib'
6
7
  require 'mogilefs'
7
8
  Hoe.plugin :seattlerb
8
9
 
10
+ manifest = "Manifest.txt"
11
+ if ! File.exist?(manifest) ||
12
+ File.stat(manifest).mtime < File.stat(RVF).mtime
13
+ system("git ls-files > #{manifest}")
14
+ File.open(manifest, "a") { |fp| fp.puts("lib/mogilefs/version.rb") }
15
+ end
16
+
9
17
  Hoe.spec 'mogilefs-client' do
10
18
  self.rubyforge_name = 'seattlerb'
11
19
  developer 'Eric Wong', 'normalperson@yhbt.net'
data/bin/mog CHANGED
@@ -14,8 +14,7 @@ if md5_trailer_nodes = ENV["MD5_TRAILER_NODES"]
14
14
  end
15
15
 
16
16
  # this is to be compatible with config files used by the Perl tools
17
- def parse_config_file!(path, overwrite = false)
18
- dest = {}
17
+ def parse_config_file!(path, dest = {})
19
18
  File.open(path).each_line do |line|
20
19
  case line
21
20
  when /^(domain|class)\s*=\s*(\S+)/
@@ -43,6 +42,7 @@ config_file = nil
43
42
  ls_l = false
44
43
  ls_h = false
45
44
  chunk = false
45
+ range = false
46
46
  test = {}
47
47
  cat = { :raw => false }
48
48
 
@@ -70,6 +70,7 @@ ARGV.options do |x|
70
70
  x.on('-h', '--human-readable',
71
71
  "print sizes in human-readable format (`ls' command)") { ls_h = true }
72
72
  x.on('--chunk', "chunk uploads (`tee' command)") { chunk = true }
73
+ x.on('--range', "stream partial uploads (`tee' command)") { range = true }
73
74
  x.separator ''
74
75
  x.on('--help', 'Show this help message.') { puts x; exit }
75
76
  x.on('--version', 'Show --version') { puts "#$0 #{MogileFS::VERSION}"; exit }
@@ -77,7 +78,7 @@ ARGV.options do |x|
77
78
  end
78
79
 
79
80
  # parse the config file specified at the command-line
80
- file_cfg = config_file ? parse_config_file!(config_file, true) : {}
81
+ file_cfg = config_file ? parse_config_file!(config_file) : {}
81
82
 
82
83
  # read environment variables, too. This Ruby API favors the term
83
84
  # "hosts", however upstream MogileFS teminology favors "trackers" instead.
@@ -188,6 +189,7 @@ begin
188
189
  puts "Key: #{key}"
189
190
  puts "Size: #{info['length']}"
190
191
  puts "Class: #{info['class']}"
192
+ checksum = info['checksum'] and puts "Checksum: #{checksum}"
191
193
  o = { :pathcount => info["devcount"] }
192
194
  mg.get_paths(key, o).each_with_index do |path,i|
193
195
  puts "URL-#{i}: #{path}"
@@ -200,49 +202,27 @@ begin
200
202
  end
201
203
  exit(ok)
202
204
  when 'tee'
203
- require 'tempfile'
205
+ abort "--range and --chunk are incompatible" if range && chunk
204
206
  dkey = ARGV.shift or raise ArgumentError, '<key>'
205
207
  ARGV.shift and raise ArgumentError, '<key>'
206
208
  cfg[:noclobber] && mg.exist?(dkey) and
207
209
  abort "`#{dkey}' already exists and -n/--no-clobber was specified"
208
210
  skip_tee = File.stat('/dev/null') == $stdout.stat
211
+ largefile = :tempfile
212
+ largefile = :content_range if range
213
+ largefile = :stream if chunk
209
214
 
210
- if chunk
211
- if skip_tee
212
- tee_obj = $stdin
213
- else
214
- tee_obj = lambda do |*args|
215
- buf = $stdin.readpartial(*args)
216
- $stdout.write(buf)
217
- buf
218
- end
219
- class << tee_obj
220
- alias readpartial call
221
- end
222
- end
223
- mg.store_file(dkey, cfg[:class], tee_obj)
224
- else # buffer input, first
225
- tmp = Tempfile.new('mog-tee')
226
- tmp.sync = true
227
-
228
- # if stdout is pointing to /dev/null, don't bother installing the filter.
229
- tee_obj = tmp
230
- unless skip_tee
231
- tee_obj = lambda do |buf|
232
- $stdout.write(buf)
233
- tmp.write(buf)
234
- end
235
- class << tee_obj
236
- alias write call
237
- end
238
- end
215
+ io = mg.new_file(dkey, :class => cfg[:class], :largefile => largefile)
216
+ begin
217
+ buf = $stdin.readpartial(16384)
239
218
  begin
240
- MogileFS.io.copy_stream($stdin, tee_obj)
241
- store_file_retry(mg, dkey, cfg[:class], tmp.path)
242
- ensure
243
- tmp.close!
244
- end
219
+ io.write(buf)
220
+ $stdout.write(buf) unless skip_tee
221
+ $stdin.readpartial(16384, buf)
222
+ end while true
223
+ rescue EOFError
245
224
  end
225
+ io.close
246
226
  when 'test'
247
227
  truth, ok = true, nil
248
228
  raise ArgumentError, "-e must be specified" unless (test.size == 1)
@@ -0,0 +1,108 @@
1
+ #!/usr/bin/env ruby
2
+ # This requires:
3
+ # * net-http-persistent RubyGem
4
+ # * Ruby 1.9.2+
5
+ # * upstream MogileFS::Server 2.45 or later
6
+ $stdout.sync = $stderr.sync = true
7
+ usage = <<EOF
8
+ Usage: #$0 -t TRACKERS"
9
+
10
+ The output of this script can be piped to awk + curl to DELETE the files:
11
+ #$0 -t TRACKERS | awk '{system("curl -XDELETE "$3)}'
12
+ EOF
13
+
14
+ require 'uri'
15
+ require 'optparse'
16
+ require 'mogilefs'
17
+ require 'net/http/persistent'
18
+ Thread.abort_on_exception = true
19
+ MogileFS::VERSION <= "3.0.0" and
20
+ abort "Upgrade mogilefs-client (to a version that distributes this script)" \
21
+ "MogileFS::Admin#each_fid is probably broken in this version"
22
+
23
+ trackers = []
24
+ ARGV.options do |x|
25
+ x.banner = usage.strip
26
+ x.separator ''
27
+ x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]',
28
+ Array, 'hostnames/IP addresses of trackers') do |args|
29
+ trackers = args
30
+ end
31
+
32
+ x.on('-h', '--help', 'Show this help message.') { puts x; exit }
33
+ x.parse!
34
+ end
35
+
36
+ adm = MogileFS::Admin.new(:hosts => trackers)
37
+ NHP = Net::HTTP::Persistent.new(File.basename($0))
38
+ client = MogileFS::MogileFS.new(:hosts => trackers, :domain => "none")
39
+
40
+ def start_perdev_thread(pfx)
41
+ todo = Queue.new
42
+ done = Queue.new
43
+ Thread.new do
44
+ while fid_path = todo.shift
45
+ path = "#{pfx}#{fid_path}"
46
+ uri = URI.parse(path)
47
+ begin
48
+ resp = NHP.request(uri, Net::HTTP::Head.new(uri.path))
49
+ done << [ path, resp ]
50
+ rescue => err
51
+ done << [ path, err ]
52
+ end
53
+ end
54
+ end
55
+ [ todo, done ]
56
+ end
57
+
58
+ def setup_devices(dev_map, adm)
59
+ hosts = {}
60
+ adm.get_hosts.each do |host|
61
+ hosts[host["hostid"]] = "http://#{host['hostip']}:#{host['http_port']}/"
62
+ end
63
+
64
+ adm.get_devices.each do |device|
65
+ pfx = hosts[device["hostid"]] + "dev#{device['devid']}"
66
+ todo, done = start_perdev_thread(pfx)
67
+ dev_map[todo] = done
68
+ end
69
+ end
70
+
71
+ def check(bad, curfid, rv)
72
+ path, resp = rv
73
+ case resp
74
+ when Net::HTTPNotFound # good
75
+ when Net::HTTPOK
76
+ bad << "#{curfid} #{resp.content_length} #{path}\n"
77
+ else
78
+ warn "E: #{resp.inspect} (#{resp.class}) #{path}"
79
+ end
80
+ end
81
+
82
+ dev_map = {}
83
+ setup_devices(dev_map, adm)
84
+ next_fid = 0
85
+ adm.each_fid do |fid|
86
+ fidid = fid["fid"]
87
+
88
+ if fidid != next_fid
89
+ (next_fid..(fidid - 1)).each do |curfid|
90
+ nfid = sprintf("%010u", curfid)
91
+ /\A(\d)(\d{3})(\d{3})(?:\d{3})\z/ =~ nfid
92
+ fid_path = "/#$1/#$2/#$3/#{nfid}.fid"
93
+ bad = []
94
+ dev_map.each_key { |todo| todo << fid_path }
95
+ dev_map.each_value { |done| check(bad, curfid, done.shift) }
96
+ next if bad.empty?
97
+
98
+ begin
99
+ info = client.file_debug(curfid)
100
+ abort "BUG: #{info.inspect} found!" if info["fid_dkey"]
101
+ rescue MogileFS::Backend::UnknownFidError
102
+ end
103
+
104
+ puts bad.join
105
+ end
106
+ end
107
+ next_fid = fidid + 1
108
+ end
data/lib/mogilefs.rb CHANGED
@@ -21,7 +21,9 @@ module MogileFS
21
21
 
22
22
  # Raised when a backend is in read-only mode
23
23
  class ReadOnlyError < Error
24
- def message; 'readonly mogilefs'; end
24
+ def message # :nodoc:
25
+ 'readonly mogilefs'
26
+ end
25
27
  end
26
28
 
27
29
  # Raised when an upload is impossible
@@ -12,7 +12,7 @@ class MogileFS::Admin < MogileFS::Client
12
12
  low = -1
13
13
  rv = 0
14
14
  begin
15
- fids = list_fids(low + 1)
15
+ fids = list_fids(low)
16
16
  fids.each { |fid| yield fid }
17
17
  rv += fids.size
18
18
  end while last = fids[-1] and low = last["fid"]
@@ -273,6 +273,7 @@ class MogileFS::Admin < MogileFS::Client
273
273
  rv
274
274
  end
275
275
 
276
+ # Clears the tracker caches. Not implemented in all versions of MogileFS
276
277
  def clear_cache
277
278
  @backend.clear_cache
278
279
  end
@@ -42,6 +42,14 @@ class MogileFS::Backend
42
42
  BACKEND_ERRORS[err_snake] = const_get(err_camel)
43
43
  end
44
44
 
45
+ def self.const_missing(name) # :nodoc:
46
+ if /Error\z/ =~ name.to_s
47
+ const_set(name, Class.new(MogileFS::Error))
48
+ else
49
+ super name
50
+ end
51
+ end
52
+
45
53
  ##
46
54
  # The last error
47
55
 
@@ -62,6 +70,7 @@ class MogileFS::Backend
62
70
 
63
71
  def initialize(args)
64
72
  @hosts = args[:hosts]
73
+ @fail_timeout = args[:fail_timeout] || 5
65
74
  raise ArgumentError, "must specify at least one host" unless @hosts
66
75
  raise ArgumentError, "must specify at least one host" if @hosts.empty?
67
76
  unless @hosts == @hosts.select { |h| h =~ /:\d+$/ } then
@@ -90,6 +99,7 @@ class MogileFS::Backend
90
99
  add_command :create_open
91
100
  add_command :create_close
92
101
  add_idempotent_command :get_paths
102
+ add_idempotent_command :noop
93
103
  add_command :delete
94
104
  add_idempotent_command :sleep
95
105
  add_command :rename
@@ -115,44 +125,6 @@ class MogileFS::Backend
115
125
  add_command :set_state
116
126
  add_command :replicate_now
117
127
 
118
- # Errors copied from MogileFS/Worker/Query.pm
119
- add_error 'dup'
120
- add_error 'after_mismatch'
121
- add_error 'bad_params'
122
- add_error 'class_exists'
123
- add_error 'class_has_files'
124
- add_error 'class_not_found'
125
- add_error 'db'
126
- add_error 'domain_has_files'
127
- add_error 'domain_exists'
128
- add_error 'domain_not_empty'
129
- add_error 'domain_not_found'
130
- add_error 'failure'
131
- add_error 'host_exists'
132
- add_error 'host_mismatch'
133
- add_error 'host_not_empty'
134
- add_error 'host_not_found'
135
- add_error 'invalid_chars'
136
- add_error 'invalid_checker_level'
137
- add_error 'invalid_mindevcount'
138
- add_error 'key_exists'
139
- add_error 'no_class'
140
- add_error 'no_devices'
141
- add_error 'no_domain'
142
- add_error 'no_host'
143
- add_error 'no_ip'
144
- add_error 'no_key'
145
- add_error 'no_port'
146
- add_error 'none_match'
147
- add_error 'plugin_aborted'
148
- add_error 'state_too_high'
149
- add_error 'size_verify_error'
150
- add_error 'unknown_command'
151
- add_error 'unknown_host'
152
- add_error 'unknown_key'
153
- add_error 'unknown_state'
154
- add_error 'unreg_domain'
155
-
156
128
  def shutdown_unlocked(do_raise = false) # :nodoc:
157
129
  @pending = []
158
130
  if @socket
@@ -254,11 +226,13 @@ class MogileFS::Backend
254
226
 
255
227
  # Performs the +cmd+ request with +args+.
256
228
  def do_request(cmd, args, idempotent = false)
257
- request = make_request cmd, args
229
+ no_raise = args.delete(:ruby_no_raise)
230
+ request = make_request(cmd, args)
258
231
  @mutex.synchronize do
259
232
  begin
260
233
  io = dispatch_unlocked(request)
261
- line = io.timed_gets(@timeout) and return parse_response(line)
234
+ line = io.timed_gets(@timeout) and
235
+ return parse_response(line, no_raise ? request : nil)
262
236
 
263
237
  idempotent or
264
238
  raise EOFError, "end of file reached after: #{request.inspect}"
@@ -349,17 +323,15 @@ class MogileFS::Backend
349
323
  def socket
350
324
  return @socket if @socket and not @socket.closed?
351
325
 
352
- now = Time.now
353
-
354
326
  @hosts.shuffle.each do |host|
355
- next if @dead.include?(host) and @dead[host][0] > now - 5
327
+ next if dead = @dead[host] and dead[0] > (Time.now - @fail_timeout)
356
328
 
357
329
  begin
358
330
  addr, port = host.split(/:/)
359
331
  @socket = MogileFS::Socket.tcp(addr, port, @timeout)
360
332
  @active_host = host
361
333
  rescue SystemCallError, MogileFS::Timeout => err
362
- @dead[host] = [ now, err ]
334
+ @dead[host] = [ Time.now, err ]
363
335
  next
364
336
  end
365
337
 
@@ -1,12 +1,12 @@
1
1
  # -*- encoding: binary -*-
2
- require "digest/md5"
3
2
  class MogileFS::Chunker
4
3
  CRLF = "\r\n"
5
4
  attr_reader :io
6
5
 
7
- def initialize(io, md5)
6
+ def initialize(io, md5, expect_md5)
8
7
  @io = io
9
- @md5 = md5 ? Digest::MD5.new : nil
8
+ @md5 = md5
9
+ @expect_md5 = expect_md5
10
10
  end
11
11
 
12
12
  def write(buf)
@@ -20,8 +20,14 @@ class MogileFS::Chunker
20
20
 
21
21
  def flush
22
22
  if @md5
23
- content_md5 = [ @md5.digest ].pack('m').strip
24
- warn "Content-MD5: #{content_md5}\r\n" if $DEBUG
23
+ content_md5 = [ @md5.digest ].pack('m').rstrip!
24
+ if @expect_md5.respond_to?(:call)
25
+ expect = @expect_md5.call.strip
26
+ if expect != content_md5
27
+ raise MogileFS::ChecksumMismatchError,
28
+ "expected: #{expect.inspect} actual: #{content_md5.inspect}"
29
+ end
30
+ end
25
31
  @io.write("0\r\nContent-MD5: #{content_md5}\r\n\r\n")
26
32
  else
27
33
  @io.write("0\r\n\r\n")