mogilefs-client 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +3 -0
- data/.gitignore +1 -0
- data/GIT-VERSION-GEN +1 -1
- data/GNUmakefile +2 -10
- data/History +3 -0
- data/Rakefile +8 -0
- data/bin/mog +18 -38
- data/examples/stale_fid_checker.rb +108 -0
- data/lib/mogilefs.rb +3 -1
- data/lib/mogilefs/admin.rb +2 -1
- data/lib/mogilefs/backend.rb +16 -44
- data/lib/mogilefs/chunker.rb +11 -5
- data/lib/mogilefs/client.rb +6 -1
- data/lib/mogilefs/http_file.rb +51 -88
- data/lib/mogilefs/mogilefs.rb +91 -39
- data/lib/mogilefs/new_file.rb +78 -0
- data/lib/mogilefs/new_file/common.rb +89 -0
- data/lib/mogilefs/new_file/content_range.rb +105 -0
- data/lib/mogilefs/new_file/stream.rb +91 -0
- data/lib/mogilefs/new_file/tempfile.rb +24 -0
- data/lib/mogilefs/new_file/writer.rb +57 -0
- data/lib/mogilefs/pool.rb +18 -10
- data/lib/mogilefs/socket/kgio.rb +4 -3
- data/lib/mogilefs/socket/pure_ruby.rb +3 -3
- data/lib/mogilefs/socket_common.rb +7 -6
- data/test/fresh.rb +5 -2
- data/test/test_backend.rb +16 -8
- data/test/test_mogilefs_integration.rb +121 -0
- data/test/test_mogstored_rack.rb +112 -6
- data/test/test_pool.rb +26 -0
- metadata +13 -10
- data/Manifest.txt +0 -60
- data/examples/mogstored_rack.rb +0 -188
- data/test/test_unit_mogstored_rack.rb +0 -72
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mogilefs-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 3
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 3
|
8
|
+
- 1
|
8
9
|
- 0
|
9
|
-
|
10
|
-
version: 3.0.0
|
10
|
+
version: 3.1.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Eric Wong
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-12-15 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: hoe
|
@@ -43,8 +43,8 @@ executables:
|
|
43
43
|
- mog
|
44
44
|
extensions: []
|
45
45
|
|
46
|
-
extra_rdoc_files:
|
47
|
-
|
46
|
+
extra_rdoc_files: []
|
47
|
+
|
48
48
|
files:
|
49
49
|
- .document
|
50
50
|
- .gitignore
|
@@ -54,12 +54,11 @@ files:
|
|
54
54
|
- HACKING
|
55
55
|
- History
|
56
56
|
- LICENSE
|
57
|
-
- Manifest.txt
|
58
57
|
- README
|
59
58
|
- Rakefile
|
60
59
|
- TODO
|
61
60
|
- bin/mog
|
62
|
-
- examples/
|
61
|
+
- examples/stale_fid_checker.rb
|
63
62
|
- lib/mogilefs.rb
|
64
63
|
- lib/mogilefs/admin.rb
|
65
64
|
- lib/mogilefs/backend.rb
|
@@ -72,6 +71,12 @@ files:
|
|
72
71
|
- lib/mogilefs/http_reader.rb
|
73
72
|
- lib/mogilefs/mogilefs.rb
|
74
73
|
- lib/mogilefs/mysql.rb
|
74
|
+
- lib/mogilefs/new_file.rb
|
75
|
+
- lib/mogilefs/new_file/common.rb
|
76
|
+
- lib/mogilefs/new_file/content_range.rb
|
77
|
+
- lib/mogilefs/new_file/stream.rb
|
78
|
+
- lib/mogilefs/new_file/tempfile.rb
|
79
|
+
- lib/mogilefs/new_file/writer.rb
|
75
80
|
- lib/mogilefs/paths_size.rb
|
76
81
|
- lib/mogilefs/pool.rb
|
77
82
|
- lib/mogilefs/socket.rb
|
@@ -104,7 +109,6 @@ files:
|
|
104
109
|
- test/test_mogtool_bigfile.rb
|
105
110
|
- test/test_mysql.rb
|
106
111
|
- test/test_pool.rb
|
107
|
-
- test/test_unit_mogstored_rack.rb
|
108
112
|
- lib/mogilefs/version.rb
|
109
113
|
- .gemtest
|
110
114
|
homepage: http://bogomips.org/mogilefs-client
|
@@ -143,7 +147,6 @@ specification_version: 3
|
|
143
147
|
summary: MogileFS client library for Ruby
|
144
148
|
test_files:
|
145
149
|
- test/test_http_reader.rb
|
146
|
-
- test/test_unit_mogstored_rack.rb
|
147
150
|
- test/test_mogilefs_integration_list_keys.rb
|
148
151
|
- test/test_db_backend.rb
|
149
152
|
- test/test_mogilefs_socket_pure.rb
|
data/Manifest.txt
DELETED
@@ -1,60 +0,0 @@
|
|
1
|
-
.document
|
2
|
-
.gitignore
|
3
|
-
.wrongdoc.yml
|
4
|
-
GIT-VERSION-GEN
|
5
|
-
GNUmakefile
|
6
|
-
HACKING
|
7
|
-
History
|
8
|
-
LICENSE
|
9
|
-
Manifest.txt
|
10
|
-
README
|
11
|
-
Rakefile
|
12
|
-
TODO
|
13
|
-
bin/mog
|
14
|
-
examples/mogstored_rack.rb
|
15
|
-
lib/mogilefs.rb
|
16
|
-
lib/mogilefs/admin.rb
|
17
|
-
lib/mogilefs/backend.rb
|
18
|
-
lib/mogilefs/bigfile.rb
|
19
|
-
lib/mogilefs/bigfile/filter.rb
|
20
|
-
lib/mogilefs/chunker.rb
|
21
|
-
lib/mogilefs/client.rb
|
22
|
-
lib/mogilefs/copy_stream.rb
|
23
|
-
lib/mogilefs/http_file.rb
|
24
|
-
lib/mogilefs/http_reader.rb
|
25
|
-
lib/mogilefs/mogilefs.rb
|
26
|
-
lib/mogilefs/mysql.rb
|
27
|
-
lib/mogilefs/paths_size.rb
|
28
|
-
lib/mogilefs/pool.rb
|
29
|
-
lib/mogilefs/socket.rb
|
30
|
-
lib/mogilefs/socket/kgio.rb
|
31
|
-
lib/mogilefs/socket/pure_ruby.rb
|
32
|
-
lib/mogilefs/socket_common.rb
|
33
|
-
lib/mogilefs/util.rb
|
34
|
-
setup.rb
|
35
|
-
test/.gitignore
|
36
|
-
test/aggregate.rb
|
37
|
-
test/exec.rb
|
38
|
-
test/fresh.rb
|
39
|
-
test/integration.rb
|
40
|
-
test/setup.rb
|
41
|
-
test/socket_test.rb
|
42
|
-
test/test_admin.rb
|
43
|
-
test/test_backend.rb
|
44
|
-
test/test_bigfile.rb
|
45
|
-
test/test_client.rb
|
46
|
-
test/test_db_backend.rb
|
47
|
-
test/test_fresh.rb
|
48
|
-
test/test_http_reader.rb
|
49
|
-
test/test_mogilefs.rb
|
50
|
-
test/test_mogilefs_integration.rb
|
51
|
-
test/test_mogilefs_integration_large_pipe.rb
|
52
|
-
test/test_mogilefs_integration_list_keys.rb
|
53
|
-
test/test_mogilefs_socket_kgio.rb
|
54
|
-
test/test_mogilefs_socket_pure.rb
|
55
|
-
test/test_mogstored_rack.rb
|
56
|
-
test/test_mogtool_bigfile.rb
|
57
|
-
test/test_mysql.rb
|
58
|
-
test/test_pool.rb
|
59
|
-
test/test_unit_mogstored_rack.rb
|
60
|
-
lib/mogilefs/version.rb
|
data/examples/mogstored_rack.rb
DELETED
@@ -1,188 +0,0 @@
|
|
1
|
-
# -*- encoding: binary -*-
|
2
|
-
require 'digest/md5'
|
3
|
-
require 'rack'
|
4
|
-
|
5
|
-
# Rack application for handling HTTP PUT/DELETE/MKCOL operations needed
|
6
|
-
# for a MogileFS storage server. GET requests are handled by
|
7
|
-
# Rack::File and Rack::Head _must_ be in the middleware stack for
|
8
|
-
# mogilefsd fsck to work properly with keepalive.
|
9
|
-
#
|
10
|
-
# Usage in rackup config file (config.ru):
|
11
|
-
#
|
12
|
-
# require "./mogstored_rack"
|
13
|
-
# use Rack::Head
|
14
|
-
# run MogstoredRack.new("/var/mogdata")
|
15
|
-
class MogstoredRack
|
16
|
-
class ContentMD5 < Digest::MD5
|
17
|
-
def content_md5
|
18
|
-
[ digest ].pack("m").strip!
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
def initialize(root, opts = {})
|
23
|
-
@root = File.expand_path(root)
|
24
|
-
@io_size = opts[:io_size] || 0x100000
|
25
|
-
@rack_file = (opts[:app] || Rack::File.new(@root))
|
26
|
-
@fsync = !! opts[:fsync]
|
27
|
-
@creat_perms = opts[:creat_perms] || (~File.umask & 0666)
|
28
|
-
@mkdir_perms = opts[:mkdir_perms] || (~File.umask & 0777)
|
29
|
-
@reread_verify = !! opts[:reread_verify]
|
30
|
-
@open_flags = opts[:open_flags] || 0
|
31
|
-
@open_flags |= IO::RDWR | IO::CREAT | IO::EXCL
|
32
|
-
end
|
33
|
-
|
34
|
-
def tmpfile(basename, dir)
|
35
|
-
t = Time.now.utc.strftime("%Y%m%d%H%M%S")
|
36
|
-
seq = 0
|
37
|
-
begin
|
38
|
-
fp = File.open("#{dir}/#{basename}.#{t}.#{seq}.tmp", @open_flags, 0600)
|
39
|
-
rescue Errno::EEXIST
|
40
|
-
retry if (seq += 1) < 10
|
41
|
-
raise
|
42
|
-
end
|
43
|
-
fp.binmode
|
44
|
-
fp.sync = true
|
45
|
-
fp
|
46
|
-
end
|
47
|
-
|
48
|
-
def call(env)
|
49
|
-
case env["REQUEST_METHOD"]
|
50
|
-
when "GET", "HEAD"
|
51
|
-
case env["PATH_INFO"]
|
52
|
-
when "/"
|
53
|
-
r(200, "") # MogileFS seems to need this...
|
54
|
-
else
|
55
|
-
@rack_file.call(env)
|
56
|
-
end
|
57
|
-
when "PUT"
|
58
|
-
put(env)
|
59
|
-
when "DELETE"
|
60
|
-
delete(env)
|
61
|
-
when "MKCOL"
|
62
|
-
mkcol(env)
|
63
|
-
else
|
64
|
-
r(405, "unsupported method", env)
|
65
|
-
end
|
66
|
-
rescue Errno::EPERM, Errno::EACCES => err
|
67
|
-
r(403, "#{err.message} (#{err.class})", env)
|
68
|
-
rescue => err
|
69
|
-
r(500, "#{err.message} (#{err.class})", env)
|
70
|
-
end
|
71
|
-
|
72
|
-
def mkcol(env)
|
73
|
-
path = server_path(env) or return r(400)
|
74
|
-
Dir.mkdir(path, @mkdir_perms)
|
75
|
-
r(204)
|
76
|
-
rescue Errno::EEXIST # succeed (204) on race condition
|
77
|
-
File.directory?(path) ? r(204) : r(409)
|
78
|
-
end
|
79
|
-
|
80
|
-
def delete(env)
|
81
|
-
path = server_path(env) or return r(400)
|
82
|
-
File.exist?(path) or return r(404)
|
83
|
-
File.directory?(path) ? Dir.rmdir(path) : File.unlink(path)
|
84
|
-
r(204)
|
85
|
-
rescue Errno::ENOENT # return 404 on race condition
|
86
|
-
File.exist?(path) ? r(500) : r(404)
|
87
|
-
end
|
88
|
-
|
89
|
-
def put(env)
|
90
|
-
path = server_path(env) or return r(400)
|
91
|
-
dir = File.dirname(path)
|
92
|
-
File.directory?(dir) or return r(403)
|
93
|
-
|
94
|
-
tmp = tmpfile(File.basename(path), dir)
|
95
|
-
buf = ""
|
96
|
-
received = put_loop(env["rack.input"], tmp, buf)
|
97
|
-
err = content_md5_fail?(env, received) and return err
|
98
|
-
if @reread_verify && err = reread_md5_fail?(env, tmp, received, buf)
|
99
|
-
return err
|
100
|
-
end
|
101
|
-
tmp.chmod(@creat_perms)
|
102
|
-
File.rename(tmp.path, path)
|
103
|
-
fsync(dir, tmp) if @fsync
|
104
|
-
resp = r(201)
|
105
|
-
resp[1]["X-Received-Content-MD5"] = received
|
106
|
-
resp
|
107
|
-
rescue
|
108
|
-
File.unlink(tmp.path) if tmp && File.exist?(tmp.path)
|
109
|
-
raise
|
110
|
-
ensure
|
111
|
-
tmp.close if tmp
|
112
|
-
end
|
113
|
-
|
114
|
-
def put_loop(src, dst, buf)
|
115
|
-
md5 = ContentMD5.new
|
116
|
-
while src.read(@io_size, buf)
|
117
|
-
md5.update(buf)
|
118
|
-
dst.write(buf)
|
119
|
-
end
|
120
|
-
md5.content_md5
|
121
|
-
end
|
122
|
-
|
123
|
-
def server_path(env)
|
124
|
-
path = env['PATH_INFO'].squeeze('/')
|
125
|
-
path.split(%r{/}).include?("..") and return false
|
126
|
-
"#@root#{path}"
|
127
|
-
end
|
128
|
-
|
129
|
-
# returns a plain-text HTTP response
|
130
|
-
def r(code, msg = nil, env = nil)
|
131
|
-
if env && logger = env["rack.logger"]
|
132
|
-
logger.warn("#{env['REQUEST_METHOD']} #{env['PATH_INFO']} " \
|
133
|
-
"#{code} #{msg.inspect}")
|
134
|
-
end
|
135
|
-
if Rack::Utils::STATUS_WITH_NO_ENTITY_BODY.include?(code)
|
136
|
-
[ code, {}, [] ]
|
137
|
-
else
|
138
|
-
msg ||= Rack::Utils::HTTP_STATUS_CODES[code] || ""
|
139
|
-
msg += "\n" if msg.size > 0
|
140
|
-
[ code,
|
141
|
-
{ 'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s },
|
142
|
-
[ msg ] ]
|
143
|
-
end
|
144
|
-
end
|
145
|
-
|
146
|
-
# Tries to detect filesystem/disk corruption.
|
147
|
-
# Unfortunately, posix_fadvise(2)/IO#advise is only advisory and
|
148
|
-
# can't guarantee we're not just reading the data in the kernel
|
149
|
-
# page cache.
|
150
|
-
def reread_md5_fail?(env, tmp, received, buf)
|
151
|
-
# try to force a reread from the storage device, not cache
|
152
|
-
tmp.fsync
|
153
|
-
tmp.rewind
|
154
|
-
tmp.advise(:dontneed) rescue nil # only in Ruby 1.9.3 and only advisory
|
155
|
-
|
156
|
-
md5 = ContentMD5.new
|
157
|
-
while tmp.read(0x4000, buf)
|
158
|
-
md5.update(buf)
|
159
|
-
end
|
160
|
-
reread = md5.content_md5
|
161
|
-
reread == received and return false # success
|
162
|
-
r(500, "reread MD5 mismatch\n" \
|
163
|
-
"received: #{received}\n" \
|
164
|
-
" reread: #{reread}", env)
|
165
|
-
end
|
166
|
-
|
167
|
-
# Tries to detect network corruption by verifying the client-supplied
|
168
|
-
# Content-MD5 is correct. It's highly unlikely the MD5 can be corrupted
|
169
|
-
# in a way that also allows corrupt data to pass through.
|
170
|
-
#
|
171
|
-
# The Rainbows!/Unicorn HTTP servers will populate the HTTP_CONTENT_MD5
|
172
|
-
# field in +env+ after env["rack.input"] is fully-consumed. Clients
|
173
|
-
# may also send Content-MD5 as a header and this will still work.
|
174
|
-
def content_md5_fail?(env, received)
|
175
|
-
expected = env["HTTP_CONTENT_MD5"] or return false
|
176
|
-
expected = expected.strip
|
177
|
-
expected == received and return false # success
|
178
|
-
r(400, "Content-MD5 mismatch\n" \
|
179
|
-
"expected: #{expected}\n" \
|
180
|
-
"received: #{received}", env)
|
181
|
-
end
|
182
|
-
|
183
|
-
# fsync each and every directory component above us on the same device
|
184
|
-
def fsync(dir, tmp)
|
185
|
-
tmp.fsync
|
186
|
-
File.open(dir) { |io| io.fsync }
|
187
|
-
end
|
188
|
-
end
|
@@ -1,72 +0,0 @@
|
|
1
|
-
# -*- encoding: binary -*-
|
2
|
-
require "test/unit"
|
3
|
-
require "tmpdir"
|
4
|
-
require "fileutils"
|
5
|
-
begin
|
6
|
-
require "./examples/mogstored_rack"
|
7
|
-
rescue LoadError
|
8
|
-
end
|
9
|
-
|
10
|
-
class TestUnitMogstoredRack < Test::Unit::TestCase
|
11
|
-
attr_reader :req
|
12
|
-
|
13
|
-
def setup
|
14
|
-
@docroot = Dir.mktmpdir(["mogstored_rack", ".docroot"])
|
15
|
-
end
|
16
|
-
|
17
|
-
def test_defaults
|
18
|
-
req = Rack::MockRequest.new(MogstoredRack.new(@docroot))
|
19
|
-
all_methods(req)
|
20
|
-
end
|
21
|
-
|
22
|
-
def test_fsync_true
|
23
|
-
req = Rack::MockRequest.new(MogstoredRack.new(@docroot, :fsync=>true))
|
24
|
-
all_methods(req)
|
25
|
-
end
|
26
|
-
|
27
|
-
def test_reread_verify
|
28
|
-
app = MogstoredRack.new(@docroot, :reread_verify=>true)
|
29
|
-
req = Rack::MockRequest.new(app)
|
30
|
-
all_methods(req)
|
31
|
-
end
|
32
|
-
|
33
|
-
def all_methods(req)
|
34
|
-
assert_equal 200, req.get("/").status
|
35
|
-
assert ! File.directory?("#@docroot/dev666")
|
36
|
-
assert_equal 204, req.request("MKCOL", "/dev666").status
|
37
|
-
assert File.directory?("#@docroot/dev666")
|
38
|
-
|
39
|
-
io = StringIO.new("HELLO")
|
40
|
-
r = req.request("PUT", "/dev666/666.fid", :input => io)
|
41
|
-
assert_equal 201, r.status
|
42
|
-
assert_equal "HELLO", IO.read("#@docroot/dev666/666.fid")
|
43
|
-
|
44
|
-
# invalid MD5
|
45
|
-
io = StringIO.new("WORLD")
|
46
|
-
md5 = [ Digest::MD5.new.digest ].pack("m").strip!
|
47
|
-
opts = { :input => io, "HTTP_CONTENT_MD5" => md5 }
|
48
|
-
r = req.request("PUT", "/dev666/666.fid", opts)
|
49
|
-
assert_equal 400, r.status
|
50
|
-
assert_equal "HELLO", IO.read("#@docroot/dev666/666.fid")
|
51
|
-
|
52
|
-
# valid MD5
|
53
|
-
io = StringIO.new("VALID")
|
54
|
-
md5 = [ Digest::MD5.digest("VALID") ].pack("m").strip!
|
55
|
-
opts = { :input => io, "HTTP_CONTENT_MD5" => md5 }
|
56
|
-
r = req.request("PUT", "/dev666/666.fid", opts)
|
57
|
-
assert_equal 201, r.status
|
58
|
-
assert_equal "VALID", IO.read("#@docroot/dev666/666.fid")
|
59
|
-
|
60
|
-
r = req.request("GET", "/dev666/666.fid")
|
61
|
-
assert_equal 200, r.status
|
62
|
-
assert_equal "VALID", r.body
|
63
|
-
|
64
|
-
r = req.request("DELETE", "/dev666/666.fid")
|
65
|
-
assert_equal 204, r.status
|
66
|
-
assert ! File.exist?("#@docroot/dev666/666.fid")
|
67
|
-
end
|
68
|
-
|
69
|
-
def teardown
|
70
|
-
FileUtils.rmtree(@docroot)
|
71
|
-
end
|
72
|
-
end if defined?(Rack)
|