mogilefs-client 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mogilefs-client
3
3
  version: !ruby/object:Gem::Version
4
- hash: 7
4
+ hash: 3
5
5
  prerelease:
6
6
  segments:
7
7
  - 3
8
+ - 1
8
9
  - 0
9
- - 0
10
- version: 3.0.0
10
+ version: 3.1.0
11
11
  platform: ruby
12
12
  authors:
13
13
  - Eric Wong
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2011-11-28 00:00:00 Z
18
+ date: 2011-12-15 00:00:00 Z
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency
21
21
  name: hoe
@@ -43,8 +43,8 @@ executables:
43
43
  - mog
44
44
  extensions: []
45
45
 
46
- extra_rdoc_files:
47
- - Manifest.txt
46
+ extra_rdoc_files: []
47
+
48
48
  files:
49
49
  - .document
50
50
  - .gitignore
@@ -54,12 +54,11 @@ files:
54
54
  - HACKING
55
55
  - History
56
56
  - LICENSE
57
- - Manifest.txt
58
57
  - README
59
58
  - Rakefile
60
59
  - TODO
61
60
  - bin/mog
62
- - examples/mogstored_rack.rb
61
+ - examples/stale_fid_checker.rb
63
62
  - lib/mogilefs.rb
64
63
  - lib/mogilefs/admin.rb
65
64
  - lib/mogilefs/backend.rb
@@ -72,6 +71,12 @@ files:
72
71
  - lib/mogilefs/http_reader.rb
73
72
  - lib/mogilefs/mogilefs.rb
74
73
  - lib/mogilefs/mysql.rb
74
+ - lib/mogilefs/new_file.rb
75
+ - lib/mogilefs/new_file/common.rb
76
+ - lib/mogilefs/new_file/content_range.rb
77
+ - lib/mogilefs/new_file/stream.rb
78
+ - lib/mogilefs/new_file/tempfile.rb
79
+ - lib/mogilefs/new_file/writer.rb
75
80
  - lib/mogilefs/paths_size.rb
76
81
  - lib/mogilefs/pool.rb
77
82
  - lib/mogilefs/socket.rb
@@ -104,7 +109,6 @@ files:
104
109
  - test/test_mogtool_bigfile.rb
105
110
  - test/test_mysql.rb
106
111
  - test/test_pool.rb
107
- - test/test_unit_mogstored_rack.rb
108
112
  - lib/mogilefs/version.rb
109
113
  - .gemtest
110
114
  homepage: http://bogomips.org/mogilefs-client
@@ -143,7 +147,6 @@ specification_version: 3
143
147
  summary: MogileFS client library for Ruby
144
148
  test_files:
145
149
  - test/test_http_reader.rb
146
- - test/test_unit_mogstored_rack.rb
147
150
  - test/test_mogilefs_integration_list_keys.rb
148
151
  - test/test_db_backend.rb
149
152
  - test/test_mogilefs_socket_pure.rb
data/Manifest.txt DELETED
@@ -1,60 +0,0 @@
1
- .document
2
- .gitignore
3
- .wrongdoc.yml
4
- GIT-VERSION-GEN
5
- GNUmakefile
6
- HACKING
7
- History
8
- LICENSE
9
- Manifest.txt
10
- README
11
- Rakefile
12
- TODO
13
- bin/mog
14
- examples/mogstored_rack.rb
15
- lib/mogilefs.rb
16
- lib/mogilefs/admin.rb
17
- lib/mogilefs/backend.rb
18
- lib/mogilefs/bigfile.rb
19
- lib/mogilefs/bigfile/filter.rb
20
- lib/mogilefs/chunker.rb
21
- lib/mogilefs/client.rb
22
- lib/mogilefs/copy_stream.rb
23
- lib/mogilefs/http_file.rb
24
- lib/mogilefs/http_reader.rb
25
- lib/mogilefs/mogilefs.rb
26
- lib/mogilefs/mysql.rb
27
- lib/mogilefs/paths_size.rb
28
- lib/mogilefs/pool.rb
29
- lib/mogilefs/socket.rb
30
- lib/mogilefs/socket/kgio.rb
31
- lib/mogilefs/socket/pure_ruby.rb
32
- lib/mogilefs/socket_common.rb
33
- lib/mogilefs/util.rb
34
- setup.rb
35
- test/.gitignore
36
- test/aggregate.rb
37
- test/exec.rb
38
- test/fresh.rb
39
- test/integration.rb
40
- test/setup.rb
41
- test/socket_test.rb
42
- test/test_admin.rb
43
- test/test_backend.rb
44
- test/test_bigfile.rb
45
- test/test_client.rb
46
- test/test_db_backend.rb
47
- test/test_fresh.rb
48
- test/test_http_reader.rb
49
- test/test_mogilefs.rb
50
- test/test_mogilefs_integration.rb
51
- test/test_mogilefs_integration_large_pipe.rb
52
- test/test_mogilefs_integration_list_keys.rb
53
- test/test_mogilefs_socket_kgio.rb
54
- test/test_mogilefs_socket_pure.rb
55
- test/test_mogstored_rack.rb
56
- test/test_mogtool_bigfile.rb
57
- test/test_mysql.rb
58
- test/test_pool.rb
59
- test/test_unit_mogstored_rack.rb
60
- lib/mogilefs/version.rb
@@ -1,188 +0,0 @@
1
- # -*- encoding: binary -*-
2
- require 'digest/md5'
3
- require 'rack'
4
-
5
- # Rack application for handling HTTP PUT/DELETE/MKCOL operations needed
6
- # for a MogileFS storage server. GET requests are handled by
7
- # Rack::File and Rack::Head _must_ be in the middleware stack for
8
- # mogilefsd fsck to work properly with keepalive.
9
- #
10
- # Usage in rackup config file (config.ru):
11
- #
12
- # require "./mogstored_rack"
13
- # use Rack::Head
14
- # run MogstoredRack.new("/var/mogdata")
15
- class MogstoredRack
16
- class ContentMD5 < Digest::MD5
17
- def content_md5
18
- [ digest ].pack("m").strip!
19
- end
20
- end
21
-
22
- def initialize(root, opts = {})
23
- @root = File.expand_path(root)
24
- @io_size = opts[:io_size] || 0x100000
25
- @rack_file = (opts[:app] || Rack::File.new(@root))
26
- @fsync = !! opts[:fsync]
27
- @creat_perms = opts[:creat_perms] || (~File.umask & 0666)
28
- @mkdir_perms = opts[:mkdir_perms] || (~File.umask & 0777)
29
- @reread_verify = !! opts[:reread_verify]
30
- @open_flags = opts[:open_flags] || 0
31
- @open_flags |= IO::RDWR | IO::CREAT | IO::EXCL
32
- end
33
-
34
- def tmpfile(basename, dir)
35
- t = Time.now.utc.strftime("%Y%m%d%H%M%S")
36
- seq = 0
37
- begin
38
- fp = File.open("#{dir}/#{basename}.#{t}.#{seq}.tmp", @open_flags, 0600)
39
- rescue Errno::EEXIST
40
- retry if (seq += 1) < 10
41
- raise
42
- end
43
- fp.binmode
44
- fp.sync = true
45
- fp
46
- end
47
-
48
- def call(env)
49
- case env["REQUEST_METHOD"]
50
- when "GET", "HEAD"
51
- case env["PATH_INFO"]
52
- when "/"
53
- r(200, "") # MogileFS seems to need this...
54
- else
55
- @rack_file.call(env)
56
- end
57
- when "PUT"
58
- put(env)
59
- when "DELETE"
60
- delete(env)
61
- when "MKCOL"
62
- mkcol(env)
63
- else
64
- r(405, "unsupported method", env)
65
- end
66
- rescue Errno::EPERM, Errno::EACCES => err
67
- r(403, "#{err.message} (#{err.class})", env)
68
- rescue => err
69
- r(500, "#{err.message} (#{err.class})", env)
70
- end
71
-
72
- def mkcol(env)
73
- path = server_path(env) or return r(400)
74
- Dir.mkdir(path, @mkdir_perms)
75
- r(204)
76
- rescue Errno::EEXIST # succeed (204) on race condition
77
- File.directory?(path) ? r(204) : r(409)
78
- end
79
-
80
- def delete(env)
81
- path = server_path(env) or return r(400)
82
- File.exist?(path) or return r(404)
83
- File.directory?(path) ? Dir.rmdir(path) : File.unlink(path)
84
- r(204)
85
- rescue Errno::ENOENT # return 404 on race condition
86
- File.exist?(path) ? r(500) : r(404)
87
- end
88
-
89
- def put(env)
90
- path = server_path(env) or return r(400)
91
- dir = File.dirname(path)
92
- File.directory?(dir) or return r(403)
93
-
94
- tmp = tmpfile(File.basename(path), dir)
95
- buf = ""
96
- received = put_loop(env["rack.input"], tmp, buf)
97
- err = content_md5_fail?(env, received) and return err
98
- if @reread_verify && err = reread_md5_fail?(env, tmp, received, buf)
99
- return err
100
- end
101
- tmp.chmod(@creat_perms)
102
- File.rename(tmp.path, path)
103
- fsync(dir, tmp) if @fsync
104
- resp = r(201)
105
- resp[1]["X-Received-Content-MD5"] = received
106
- resp
107
- rescue
108
- File.unlink(tmp.path) if tmp && File.exist?(tmp.path)
109
- raise
110
- ensure
111
- tmp.close if tmp
112
- end
113
-
114
- def put_loop(src, dst, buf)
115
- md5 = ContentMD5.new
116
- while src.read(@io_size, buf)
117
- md5.update(buf)
118
- dst.write(buf)
119
- end
120
- md5.content_md5
121
- end
122
-
123
- def server_path(env)
124
- path = env['PATH_INFO'].squeeze('/')
125
- path.split(%r{/}).include?("..") and return false
126
- "#@root#{path}"
127
- end
128
-
129
- # returns a plain-text HTTP response
130
- def r(code, msg = nil, env = nil)
131
- if env && logger = env["rack.logger"]
132
- logger.warn("#{env['REQUEST_METHOD']} #{env['PATH_INFO']} " \
133
- "#{code} #{msg.inspect}")
134
- end
135
- if Rack::Utils::STATUS_WITH_NO_ENTITY_BODY.include?(code)
136
- [ code, {}, [] ]
137
- else
138
- msg ||= Rack::Utils::HTTP_STATUS_CODES[code] || ""
139
- msg += "\n" if msg.size > 0
140
- [ code,
141
- { 'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s },
142
- [ msg ] ]
143
- end
144
- end
145
-
146
- # Tries to detect filesystem/disk corruption.
147
- # Unfortunately, posix_fadvise(2)/IO#advise is only advisory and
148
- # can't guarantee we're not just reading the data in the kernel
149
- # page cache.
150
- def reread_md5_fail?(env, tmp, received, buf)
151
- # try to force a reread from the storage device, not cache
152
- tmp.fsync
153
- tmp.rewind
154
- tmp.advise(:dontneed) rescue nil # only in Ruby 1.9.3 and only advisory
155
-
156
- md5 = ContentMD5.new
157
- while tmp.read(0x4000, buf)
158
- md5.update(buf)
159
- end
160
- reread = md5.content_md5
161
- reread == received and return false # success
162
- r(500, "reread MD5 mismatch\n" \
163
- "received: #{received}\n" \
164
- " reread: #{reread}", env)
165
- end
166
-
167
- # Tries to detect network corruption by verifying the client-supplied
168
- # Content-MD5 is correct. It's highly unlikely the MD5 can be corrupted
169
- # in a way that also allows corrupt data to pass through.
170
- #
171
- # The Rainbows!/Unicorn HTTP servers will populate the HTTP_CONTENT_MD5
172
- # field in +env+ after env["rack.input"] is fully-consumed. Clients
173
- # may also send Content-MD5 as a header and this will still work.
174
- def content_md5_fail?(env, received)
175
- expected = env["HTTP_CONTENT_MD5"] or return false
176
- expected = expected.strip
177
- expected == received and return false # success
178
- r(400, "Content-MD5 mismatch\n" \
179
- "expected: #{expected}\n" \
180
- "received: #{received}", env)
181
- end
182
-
183
- # fsync each and every directory component above us on the same device
184
- def fsync(dir, tmp)
185
- tmp.fsync
186
- File.open(dir) { |io| io.fsync }
187
- end
188
- end
@@ -1,72 +0,0 @@
1
- # -*- encoding: binary -*-
2
- require "test/unit"
3
- require "tmpdir"
4
- require "fileutils"
5
- begin
6
- require "./examples/mogstored_rack"
7
- rescue LoadError
8
- end
9
-
10
- class TestUnitMogstoredRack < Test::Unit::TestCase
11
- attr_reader :req
12
-
13
- def setup
14
- @docroot = Dir.mktmpdir(["mogstored_rack", ".docroot"])
15
- end
16
-
17
- def test_defaults
18
- req = Rack::MockRequest.new(MogstoredRack.new(@docroot))
19
- all_methods(req)
20
- end
21
-
22
- def test_fsync_true
23
- req = Rack::MockRequest.new(MogstoredRack.new(@docroot, :fsync=>true))
24
- all_methods(req)
25
- end
26
-
27
- def test_reread_verify
28
- app = MogstoredRack.new(@docroot, :reread_verify=>true)
29
- req = Rack::MockRequest.new(app)
30
- all_methods(req)
31
- end
32
-
33
- def all_methods(req)
34
- assert_equal 200, req.get("/").status
35
- assert ! File.directory?("#@docroot/dev666")
36
- assert_equal 204, req.request("MKCOL", "/dev666").status
37
- assert File.directory?("#@docroot/dev666")
38
-
39
- io = StringIO.new("HELLO")
40
- r = req.request("PUT", "/dev666/666.fid", :input => io)
41
- assert_equal 201, r.status
42
- assert_equal "HELLO", IO.read("#@docroot/dev666/666.fid")
43
-
44
- # invalid MD5
45
- io = StringIO.new("WORLD")
46
- md5 = [ Digest::MD5.new.digest ].pack("m").strip!
47
- opts = { :input => io, "HTTP_CONTENT_MD5" => md5 }
48
- r = req.request("PUT", "/dev666/666.fid", opts)
49
- assert_equal 400, r.status
50
- assert_equal "HELLO", IO.read("#@docroot/dev666/666.fid")
51
-
52
- # valid MD5
53
- io = StringIO.new("VALID")
54
- md5 = [ Digest::MD5.digest("VALID") ].pack("m").strip!
55
- opts = { :input => io, "HTTP_CONTENT_MD5" => md5 }
56
- r = req.request("PUT", "/dev666/666.fid", opts)
57
- assert_equal 201, r.status
58
- assert_equal "VALID", IO.read("#@docroot/dev666/666.fid")
59
-
60
- r = req.request("GET", "/dev666/666.fid")
61
- assert_equal 200, r.status
62
- assert_equal "VALID", r.body
63
-
64
- r = req.request("DELETE", "/dev666/666.fid")
65
- assert_equal 204, r.status
66
- assert ! File.exist?("#@docroot/dev666/666.fid")
67
- end
68
-
69
- def teardown
70
- FileUtils.rmtree(@docroot)
71
- end
72
- end if defined?(Rack)