mogilefs-client 1.3.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,2 @@
1
+ /pkg
2
+ /doc
data/GNUmakefile ADDED
@@ -0,0 +1,32 @@
1
+ # use GNU Make to run tests in parallel, and without depending on Rubygems
2
+ all:: test
3
+
4
+ T := $(wildcard test/test*.rb)
5
+ TO := $(subst .rb,.log,$(T))
6
+
7
+ test: $(T)
8
+ @cat $(TO) | ruby test/aggregate.rb
9
+ @$(RM) $(TO)
10
+ clean:
11
+ $(RM) $(TO) $(addsuffix +,$(TO))
12
+
13
+ t = $(basename $(notdir $@))
14
+ t_log = $(subst .rb,.log,$@)
15
+
16
+ $(T):
17
+ @echo $(t); ruby -I lib $@ $(TEST_OPTS) > $(t_log)+ 2>&1
18
+ @mv $(t_log)+ $(t_log)
19
+
20
+ # using make instead of rake since Rakefile takes too long to load
21
+ Manifest.txt:
22
+ git ls-files > $@+
23
+ mv $@+ $@
24
+
25
+ libs := $(wildcard lib/*.rb lib/*/*.rb)
26
+ flay_flags =
27
+ flog_flags =
28
+ flay: $(libs)
29
+ flay $(flay_flags) $^
30
+ flog: $(libs)
31
+ flog $(flog_flags) $^
32
+ .PHONY: $(T) Manifest.txt
data/History.txt CHANGED
@@ -1,3 +1,26 @@
1
+ = 2.0.0
2
+
3
+ * use a set of standard exceptions based on MogileFS::Error,
4
+ this is an incompatible API change (hence the 2.0.0 version number).
5
+ * remove NFS support since it's gone in MogileFS 2.x and NFS is horrible
6
+ * mog timeouts and retries increased
7
+ * more consistent handling of bad sockets, all sockets used internally
8
+ are now explicitly non-blocking and IO.select is used for timeouts
9
+ instead of using threads behind our backs
10
+ * remove open-uri and net/http dependencies, they were bad with large files
11
+ * add paths_size() method, allowing single mogilefsd call to get size and paths
12
+ * add read-only MogileFS::Mysql driver, allowing mogilefsd to be bypassed
13
+ * use TCP_NODELAY when available on sockets that require low latency
14
+ * use TCP_CORK on bulk transfers to improve bandwidth usage
15
+ * better HTTP error handling
16
+ * verify_uris method in new MogileFS::Network module which allows
17
+ async verification of several URIs at once on the client side.
18
+ * handle multiple device failover correctly on HTTP uploads
19
+ * initial big_file read support (should be mogtool(1)-compatible)
20
+ * unit tests can be run in parallel using GNU Make, 3x faster on a Core2 Duo
21
+ * unit tests modified to use real sockets for easier verification of
22
+ timeout and error condition handling.
23
+
1
24
  = 1.3.1
2
25
 
3
26
  * Fix missing MogileFS::Util include for sysrwloop in MogileFS::MogileFS
data/LICENSE.txt CHANGED
@@ -1,4 +1,5 @@
1
1
  Copyright 2005 Eric Hodel, The Robot Co-op. All rights reserved.
2
+ Copyright 2008 Eric Wong. All rights reserved.
2
3
 
3
4
  Redistribution and use in source and binary forms, with or without
4
5
  modification, are permitted provided that the following conditions
data/Manifest.txt CHANGED
@@ -1,3 +1,5 @@
1
+ .gitignore
2
+ GNUmakefile
1
3
  History.txt
2
4
  LICENSE.txt
3
5
  Manifest.txt
@@ -7,15 +9,24 @@ bin/mog
7
9
  lib/mogilefs.rb
8
10
  lib/mogilefs/admin.rb
9
11
  lib/mogilefs/backend.rb
12
+ lib/mogilefs/bigfile.rb
10
13
  lib/mogilefs/client.rb
11
14
  lib/mogilefs/httpfile.rb
12
15
  lib/mogilefs/mogilefs.rb
13
- lib/mogilefs/nfsfile.rb
16
+ lib/mogilefs/mysql.rb
17
+ lib/mogilefs/network.rb
14
18
  lib/mogilefs/pool.rb
15
19
  lib/mogilefs/util.rb
20
+ test/.gitignore
21
+ test/aggregate.rb
16
22
  test/setup.rb
17
23
  test/test_admin.rb
18
24
  test/test_backend.rb
25
+ test/test_bigfile.rb
19
26
  test/test_client.rb
27
+ test/test_db_backend.rb
20
28
  test/test_mogilefs.rb
29
+ test/test_mysql.rb
30
+ test/test_network.rb
21
31
  test/test_pool.rb
32
+ test/test_util.rb
data/README.txt CHANGED
@@ -14,10 +14,20 @@ File bugs:
14
14
 
15
15
  http://rubyforge.org/tracker/?func=add&group_id=1513&atid=5921
16
16
 
17
+ Source repository (git):
18
+
19
+ git://git.bogomips.org/mogilefs-client.git
20
+
21
+ http://git.bogomips.org/mogilefs-client.git
22
+
23
+ Repository browser (cgit):
24
+
25
+ http://git.bogomips.org/cgit/mogilefs-client.git
26
+
17
27
  == About
18
28
 
19
29
  A Ruby MogileFS client. MogileFS is a distributed filesystem written
20
- by Danga Interactive. This client supports NFS and HTTP modes.
30
+ by Danga Interactive. This client only supports HTTP.
21
31
 
22
32
  For information on MogileFS see:
23
33
 
@@ -25,7 +35,8 @@ http://danga.com/mogilefs/
25
35
 
26
36
  == Installing mogilefs-client
27
37
 
28
- First you need a MogileFS setup. You can find information on how to do that at the above URL.
38
+ First you need a MogileFS setup. You can find information on how to do
39
+ that at the above URL.
29
40
 
30
41
  Then install the gem:
31
42
 
@@ -35,32 +46,30 @@ Then install the gem:
35
46
 
36
47
  # Create a new instance that will communicate with these trackers:
37
48
  hosts = %w[192.168.1.69:6001 192.168.1.70:6001]
38
- mg = MogileFS::MogileFS.new(:domain => 'test', :hosts => hosts
39
- :root => '/mnt/mogilefs')
40
-
49
+ mg = MogileFS::MogileFS.new(:domain => 'test', :hosts => hosts)
50
+
41
51
  # Stores "A bunch of text to store" into 'some_key' with a class of 'text'.
42
52
  mg.store_content 'some_key', 'text', "A bunch of text to store"
43
-
53
+
44
54
  # Retrieve data from 'some_key'
45
55
  data = mg.get_file_data 'some_key'
46
-
56
+
47
57
  # Store the contents of 'image.jpeg' into the key 'my_image' with a class of
48
58
  # 'image'.
49
59
  mg.store_file 'my_image', 'image', 'image.jpeg'
50
-
60
+
51
61
  # Store the contents of 'image.jpeg' into the key 'my_image' with a class of
52
62
  # 'image' using an open IO.
53
63
  File.open 'image.jpeg' do |fp|
54
64
  mg.store_file 'my_image', 'image', fp
55
65
  end
56
-
66
+
57
67
  # Remove the key 'my_image' and 'some_key'.
58
68
  mg.delete 'my_image'
59
69
  mg.delete 'some_key'
60
70
 
61
71
  == WARNING!
62
72
 
63
- This client is only known to work in NFS mode. HTTP mode is implemented but
64
- only lightly tested in production environments. If you find a bug,
65
- please report it on the Rubyforge project.
66
-
73
+ This client is only supported in HTTP mode. NFS mode was previously
74
+ supported in 1.3.x, but since MogileFS 2.x has dropped support for
75
+ NFS, this client has removed support for it.
data/bin/mog CHANGED
@@ -14,6 +14,8 @@ def parse_config_file!(path, overwrite = false)
14
14
  dest[$1.to_sym] = $2
15
15
  elsif m = /^(?:trackers|hosts)\s*=\s*(.*)/.match(line)
16
16
  dest[:hosts] = $1.split(/\s*,\s*/)
17
+ elsif m = /^timeout\s*=\s*(.*)/.match(line)
18
+ dest[:timeout] = m[1].to_f
17
19
  else
18
20
  STDERR.puts "Ignored configuration line: #{line}" unless /^#/.match(line)
19
21
  end
@@ -31,6 +33,7 @@ config_file = nil
31
33
  ls_l = false
32
34
  ls_h = false
33
35
  test = {}
36
+ cat = { :raw => false }
34
37
 
35
38
  ARGV.options do |x|
36
39
  x.banner = "Usage: #{$0} [options] <command> [<arguments>]"
@@ -45,6 +48,7 @@ ARGV.options do |x|
45
48
  end
46
49
 
47
50
  x.on('-e', 'True if key exists') { test[:e] = true }
51
+ x.on('-r', '--raw', 'show raw big_info file information') { cat[:raw] = true }
48
52
 
49
53
  x.on('-C', '--class=s', 'class') { |klass| cli_cfg[:class] = klass }
50
54
  x.on('-d', '--domain=s', 'domain') { |domain| cli_cfg[:domain] = domain }
@@ -67,7 +71,7 @@ env_cfg = {}
67
71
  if ENV["MOG_TRACKERS"]
68
72
  env_cfg[:hosts] = ENV["MOG_TRACKERS"].split(/\s*,\s*/)
69
73
  end
70
- if ENV["MOG_HOSTS"] && env_cfg[:hosts].empty?
74
+ if ENV["MOG_HOSTS"] && (env_cfg[:hosts] || []).empty?
71
75
  env_cfg[:hosts] = ENV["MOG_HOSTS"].split(/\s*,\s*/)
72
76
  end
73
77
  env_cfg[:domain] = ENV["MOG_DOMAIN"] if ENV["MOG_DOMAIN"]
@@ -91,20 +95,42 @@ unless cmd = ARGV.shift
91
95
  exit 1
92
96
  end
93
97
 
98
+ cfg[:timeout] ||= 30 # longer timeout for interactive use
94
99
  include MogileFS::Util
95
100
  mg = MogileFS::MogileFS.new(cfg)
96
101
 
102
+ def store_file_retry(mg, key, storage_class, filepath)
103
+ tries = 0
104
+ begin
105
+ mg.store_file(key, storage_class, filepath)
106
+ rescue MogileFS::UnreadableSocketError,
107
+ MogileFS::Backend::NoDevicesError => err
108
+ if ((tries += 1) < 10)
109
+ STDERR.puts "Retrying on error: #{err}: #{err.message} tries: #{tries}"
110
+ retry
111
+ else
112
+ STDERR.puts "FATAL: #{err}: #{err.message} tries: #{tries}"
113
+ end
114
+ exit 1
115
+ end
116
+ end
117
+
97
118
  begin
98
119
  case cmd
99
120
  when 'cp'
100
121
  filename = ARGV.shift or raise ArgumentError, '<filename> <key>'
101
122
  key = ARGV.shift or raise ArgumentError, '<filename> <key>'
102
123
  ARGV.shift and raise ArgumentError, '<filename> <key>'
103
- cfg[:class] or raise ArgumentError, 'E: --class must be specified'
104
- mg.store_file(key, cfg[:class], filename)
124
+ store_file_retry(mg, key, cfg[:class], filename)
105
125
  when 'cat'
106
126
  ARGV.empty? and raise ArgumentError, '<key1> [<key2> ...]'
107
- ARGV.each { |key| mg.get_file_data(key) { |fp| sysrwloop(fp, STDOUT) } }
127
+ ARGV.each do |key|
128
+ if (!cat[:raw] && key =~ /^_big_info:/)
129
+ mg.bigfile_write(key, STDOUT, {:verify => true})
130
+ else
131
+ mg.get_file_data(key) { |fp| sysrwloop(fp, STDOUT) }
132
+ end
133
+ end
108
134
  when 'ls'
109
135
  prefixes = ARGV.empty? ? [ nil ] : ARGV
110
136
  prefixes.each do |prefix|
@@ -160,9 +186,14 @@ begin
160
186
  buf = ''
161
187
  tmp = Tempfile.new('mog-tee') # TODO: explore Transfer-Encoding:chunked :)
162
188
  at_exit { tmp.unlink }
189
+
190
+ # if stdout is pointing to /dev/null, don't bother installing the filter.
191
+ STDOUT.sync = true
192
+ tee_filter = File.stat('/dev/null') == STDOUT.stat ?
193
+ nil : Proc.new { |buf| STDOUT.write(buf); buf }
163
194
  begin
164
- sysrwloop(STDIN, tmp)
165
- mg.store_file(key, cfg[:class], tmp.path)
195
+ sysrwloop(STDIN, tmp, tee_filter)
196
+ store_file_retry(mg, key, cfg[:class], tmp.path)
166
197
  ensure
167
198
  tmp.close
168
199
  end
data/lib/mogilefs.rb CHANGED
@@ -6,21 +6,35 @@
6
6
 
7
7
  module MogileFS
8
8
 
9
- VERSION = '1.3.1'
9
+ VERSION = '2.0.0'.freeze
10
10
 
11
11
  ##
12
12
  # Raised when a socket remains unreadable for too long.
13
13
 
14
- class UnreadableSocketError < RuntimeError; end
14
+ class Error < StandardError; end
15
+ class UnreadableSocketError < Error; end
16
+ class SizeMismatchError < Error; end
17
+ class ChecksumMismatchError < RuntimeError; end
18
+ class ReadOnlyError < Error
19
+ def message; 'readonly mogilefs'; end
20
+ end
21
+ class EmptyPathError < Error
22
+ def message; 'Empty path for mogile upload'; end
23
+ end
24
+
25
+ class UnsupportedPathError < Error; end
26
+ class RequestTruncatedError < Error; end
27
+ class InvalidResponseError < Error; end
28
+ class UnreachableBackendError < Error
29
+ def message; "couldn't connect to mogilefsd backend"; end
30
+ end
15
31
 
16
32
  end
17
33
 
18
- require 'socket'
19
-
20
34
  require 'mogilefs/backend'
21
- require 'mogilefs/nfsfile'
22
35
  require 'mogilefs/httpfile'
23
36
  require 'mogilefs/client'
37
+ require 'mogilefs/bigfile'
24
38
  require 'mogilefs/mogilefs'
25
39
  require 'mogilefs/admin'
26
40
 
@@ -40,9 +40,8 @@ class MogileFS::Admin < MogileFS::Client
40
40
  # "altmask"=>""}]
41
41
 
42
42
  def get_hosts(hostid = nil)
43
- args = hostid ? { :hostid => hostid } : {}
44
- res = @backend.get_hosts args
45
- return clean('hosts', 'host', res)
43
+ clean('hosts', 'host',
44
+ @backend.get_hosts(hostid ? { :hostid => hostid } : {}))
46
45
  end
47
46
 
48
47
  ##
@@ -62,9 +61,8 @@ class MogileFS::Admin < MogileFS::Client
62
61
  # "mb_total"=>""}]
63
62
 
64
63
  def get_devices(devid = nil)
65
- args = devid ? { :devid => devid } : {}
66
- res = @backend.get_devices args
67
- return clean('devices', 'dev', res)
64
+ clean('devices', 'dev',
65
+ @backend.get_devices(devid ? { :devid => devid } : {}))
68
66
  end
69
67
 
70
68
  ##
@@ -88,8 +86,8 @@ class MogileFS::Admin < MogileFS::Client
88
86
  # "key"=>"new_new_key"}]
89
87
 
90
88
  def list_fids(from_fid, to_fid)
91
- res = @backend.list_fids :from => from_fid, :to => to_fid
92
- return clean('fid_count', 'fid_', res)
89
+ clean('fid_count', 'fid_',
90
+ @backend.list_fids(:from => from_fid, :to => to_fid))
93
91
  end
94
92
 
95
93
  ##
@@ -126,7 +124,7 @@ class MogileFS::Admin < MogileFS::Client
126
124
  stats.delete 'file' if stats['file'].empty?
127
125
  stats.delete 'replication' if stats['replication'].empty?
128
126
 
129
- return stats
127
+ stats
130
128
  end
131
129
 
132
130
  ##
@@ -148,25 +146,24 @@ class MogileFS::Admin < MogileFS::Client
148
146
  domains[res["domain#{i}"]] = Hash[*domain.flatten]
149
147
  end
150
148
 
151
- return domains
149
+ domains
152
150
  end
153
151
 
154
152
  ##
155
153
  # Creates a new domain named +domain+. Returns nil if creation failed.
156
154
 
157
155
  def create_domain(domain)
158
- raise 'readonly mogilefs' if readonly?
156
+ raise MogileFS::ReadOnlyError if readonly?
159
157
  res = @backend.create_domain :domain => domain
160
- return res['domain'] unless res.nil?
158
+ res ? res['domain'] : nil
161
159
  end
162
160
 
163
161
  ##
164
162
  # Deletes +domain+. Returns true if successful, false if not.
165
163
 
166
164
  def delete_domain(domain)
167
- raise 'readonly mogilefs' if readonly?
168
- res = @backend.delete_domain :domain => domain
169
- return !res.nil?
165
+ raise MogileFS::ReadOnlyError if readonly?
166
+ ! @backend.delete_domain(:domain => domain).nil?
170
167
  end
171
168
 
172
169
  ##
@@ -174,7 +171,7 @@ class MogileFS::Admin < MogileFS::Client
174
171
  # +mindevcount+ devices. Returns nil on failure.
175
172
 
176
173
  def create_class(domain, klass, mindevcount)
177
- return modify_class(domain, klass, mindevcount, :create)
174
+ modify_class(domain, klass, mindevcount, :create)
178
175
  end
179
176
 
180
177
  ##
@@ -182,7 +179,7 @@ class MogileFS::Admin < MogileFS::Client
182
179
  # devices. Returns nil on failure.
183
180
 
184
181
  def update_class(domain, klass, mindevcount)
185
- return modify_class(domain, klass, mindevcount, :update)
182
+ modify_class(domain, klass, mindevcount, :update)
186
183
  end
187
184
 
188
185
  ##
@@ -190,8 +187,7 @@ class MogileFS::Admin < MogileFS::Client
190
187
  # not.
191
188
 
192
189
  def delete_class(domain, klass)
193
- res = @backend.delete_class :domain => domain, :class => klass
194
- return !res.nil?
190
+ ! @backend.delete_class(:domain => domain, :class => klass).nil?
195
191
  end
196
192
 
197
193
  ##
@@ -202,23 +198,22 @@ class MogileFS::Admin < MogileFS::Client
202
198
  raise ArgumentError, "Must specify ip and port" unless \
203
199
  args.include? :ip and args.include? :port
204
200
 
205
- return modify_host(host, args, 'create')
201
+ modify_host(host, args, 'create')
206
202
  end
207
203
 
208
204
  ##
209
205
  # Updates +host+ with +args+. Returns true if successful, false if not.
210
206
 
211
207
  def update_host(host, args = {})
212
- return modify_host(host, args, 'update')
208
+ modify_host(host, args, 'update')
213
209
  end
214
210
 
215
211
  ##
216
212
  # Deletes host +host+. Returns nil on failure.
217
213
 
218
214
  def delete_host(host)
219
- raise 'readonly mogilefs' if readonly?
220
- res = @backend.delete_host :host => host
221
- return !res.nil?
215
+ raise MogileFS::ReadOnlyError if readonly?
216
+ ! @backend.delete_host(:host => host).nil?
222
217
  end
223
218
 
224
219
  ##
@@ -226,9 +221,8 @@ class MogileFS::Admin < MogileFS::Client
226
221
  # 'alive', 'down', or 'dead'.
227
222
 
228
223
  def change_device_state(host, device, state)
229
- raise 'readonly mogilefs' if readonly?
230
- res = @backend.set_state :host => host, :device => device, :state => state
231
- return !res.nil?
224
+ raise MogileFS::ReadOnlyError if readonly?
225
+ ! @backend.set_state(:host => host, :device => device, :state => state).nil?
232
226
  end
233
227
 
234
228
  protected unless defined? $TESTING
@@ -238,11 +232,11 @@ class MogileFS::Admin < MogileFS::Client
238
232
  # +action+. Returns the class name if successful, nil if not.
239
233
 
240
234
  def modify_class(domain, klass, mindevcount, action)
241
- raise 'readonly mogilefs' if readonly?
235
+ raise MogileFS::ReadOnlyError if readonly?
242
236
  res = @backend.send("#{action}_class", :domain => domain, :class => klass,
243
237
  :mindevcount => mindevcount)
244
238
 
245
- return res['class'] unless res.nil?
239
+ res ? res['class'] : nil
246
240
  end
247
241
 
248
242
  ##
@@ -251,8 +245,7 @@ class MogileFS::Admin < MogileFS::Client
251
245
 
252
246
  def modify_host(host, args = {}, action = 'create')
253
247
  args[:host] = host
254
- res = @backend.send "#{action}_host", args
255
- return !res.nil?
248
+ ! @backend.send("#{action}_host", args).nil?
256
249
  end
257
250
 
258
251
  ##
@@ -271,9 +264,9 @@ class MogileFS::Admin < MogileFS::Client
271
264
  # "host1_status"=>"alive",
272
265
  # "host1_altmask"=>""}
273
266
  # admin.clean 'hosts', 'host', res
274
- #
267
+ #
275
268
  # Returns:
276
- #
269
+ #
277
270
  # [{"status"=>"alive",
278
271
  # "http_get_port"=>"",
279
272
  # "http_port"=>"",
@@ -286,7 +279,7 @@ class MogileFS::Admin < MogileFS::Client
286
279
 
287
280
  def clean(count, prefix, res, underscore = true)
288
281
  underscore = underscore ? '_' : ''
289
- return (1..res[count].to_i).map do |i|
282
+ (1..res[count].to_i).map do |i|
290
283
  dev = res.select { |k,_| k =~ /^#{prefix}#{i}#{underscore}/ }.map do |k,v|
291
284
  [k.sub(/^#{prefix}#{i}#{underscore}/, ''), v]
292
285
  end