omgf 0.0.0.GIT → 0.0.1.GIT
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/GIT-VERSION-GEN +26 -36
- data/GNUmakefile +1 -0
- data/examples/verify_paths_manual_test.rb +20 -0
- data/lib/omgf.rb +1 -0
- data/lib/omgf/hysterical_raisins.rb +48 -10
- data/lib/omgf/pool.rb +2 -2
- data/lib/omgf/verify_paths.rb +241 -0
- data/omgf.gemspec +2 -0
- data/test/integration.rb +4 -4
- data/test/test_hysterical_raisins.rb +35 -7
- data/test/test_hysterical_raisins_cmogstored.rb +54 -0
- metadata +52 -2
data/.gitignore
CHANGED
data/GIT-VERSION-GEN
CHANGED
@@ -1,40 +1,30 @@
|
|
1
|
-
#!/bin/
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
'
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
CONSTANT = "OMGF::VERSION"
|
3
|
+
RVF = "lib/omgf/version.rb"
|
4
|
+
GVF = "GIT-VERSION-FILE"
|
5
|
+
DEF_VER = "v0.0.1.GIT"
|
6
|
+
vn = DEF_VER
|
8
7
|
|
9
8
|
# First see if there is a version file (included in release tarballs),
|
10
9
|
# then try git-describe, then default.
|
11
|
-
if
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
then
|
24
|
-
VN=$(echo "$VN" | sed -e 's/-/./g');
|
25
|
-
else
|
26
|
-
VN="$DEF_VER"
|
27
|
-
fi
|
28
|
-
|
29
|
-
VN=$(expr "$VN" : v*'\(.*\)')
|
10
|
+
if File.exist?(".git")
|
11
|
+
describe = `git describe --abbrev=4 HEAD 2>/dev/null`.strip
|
12
|
+
case describe
|
13
|
+
when /\Av[0-9]*/
|
14
|
+
vn = describe
|
15
|
+
system(*%w(git update-index -q --refresh))
|
16
|
+
unless `git diff-index --name-only HEAD --`.chomp.empty?
|
17
|
+
vn << "-dirty"
|
18
|
+
end
|
19
|
+
vn.tr!('-', '.')
|
20
|
+
end
|
21
|
+
end
|
30
22
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
echo "GIT_VERSION = $VN" >$GVF
|
40
|
-
}
|
23
|
+
vn = vn.sub!(/\Av/, "")
|
24
|
+
new_ruby_version = "#{CONSTANT} = '#{vn}'\n"
|
25
|
+
cur_ruby_version = File.read(RVF) rescue nil
|
26
|
+
if new_ruby_version != cur_ruby_version
|
27
|
+
File.open(GVF, "w") { |fp| fp.write("GIT_VERSION = #{vn}\n") }
|
28
|
+
File.open(RVF, "w") { |fp| fp.write(new_ruby_version) }
|
29
|
+
end
|
30
|
+
puts vn if $0 == __FILE__
|
data/GNUmakefile
CHANGED
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
require "omgf/verify_paths"
|
3
|
+
require "logger"
|
4
|
+
require "uri"
|
5
|
+
require "pp"
|
6
|
+
vp = OMGF::VerifyPaths.new(Logger.new($stderr))
|
7
|
+
urls = %w(
|
8
|
+
http://yhbt.net/
|
9
|
+
http://bogomips.org/omgf.git
|
10
|
+
http://bogomips.org/ozZZZZmgf.git
|
11
|
+
http://bogomips.org/ozZZZZ
|
12
|
+
http://bogomips.org/
|
13
|
+
http://127.0.0.1:666/
|
14
|
+
)
|
15
|
+
uris = urls.map { |uri| URI(uri) }
|
16
|
+
|
17
|
+
pp vp.verify(uris, 1, 0.1)
|
18
|
+
uris = urls.map { |uri| URI(uri) }
|
19
|
+
sleep 1
|
20
|
+
pp vp.verify(uris, 1, 0.1)
|
data/lib/omgf.rb
CHANGED
@@ -4,6 +4,7 @@
|
|
4
4
|
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
5
5
|
# :startdoc:
|
6
6
|
require "omgf/pool"
|
7
|
+
require "omgf/verify_paths"
|
7
8
|
require "time"
|
8
9
|
require "json"
|
9
10
|
require "mogilefs"
|
@@ -41,7 +42,15 @@ class OMGF::HystericalRaisins
|
|
41
42
|
noverify: opts[:noverify],
|
42
43
|
pathcount: opts[:pathcount] || 0x7fffffff,
|
43
44
|
}
|
45
|
+
@new_file_opts = {
|
46
|
+
content_md5: opts[:content_md5], # :trailer is acceptable here
|
47
|
+
|
48
|
+
# largefile: auto-selects based on env["CONTENT_LENGTH"]
|
49
|
+
largefile: opts[:largefile] || :stream,
|
50
|
+
}
|
44
51
|
@put_overwrite_header = opts[:put_overwrite_header] || "HTTP_X_OMGF_FORCE"
|
52
|
+
@vp = OMGF::VerifyPaths.new(opts[:logger])
|
53
|
+
@verify_timeout = opts[:verify_timeout] || 0.5
|
45
54
|
pool_init(mg_opts)
|
46
55
|
end
|
47
56
|
|
@@ -98,13 +107,16 @@ class OMGF::HystericalRaisins
|
|
98
107
|
|
99
108
|
# HEAD /$DOMAIN/$KEY
|
100
109
|
def stat_key(env, domain, key)
|
101
|
-
size,
|
102
|
-
|
110
|
+
size, uris = mg_size_and_uris(domain, key, @get_paths_opts)
|
111
|
+
uris = @vp.verify(uris, 1, @verify_timeout).flatten!
|
112
|
+
|
113
|
+
return r(503, "") unless uris && uris[0]
|
114
|
+
|
103
115
|
h = { "Content-Length" => size.to_s }
|
104
116
|
fn = filename(h, query(env)) || key
|
105
117
|
h["Content-Type"] = key_mime_type(fn)
|
106
|
-
unless reproxy?(env, key, h,
|
107
|
-
|
118
|
+
unless reproxy?(env, key, h, uris[0].to_s)
|
119
|
+
uris.each_with_index { |uri,i| h["X-URL-#{i}"] = uri.to_s }
|
108
120
|
end
|
109
121
|
[ 200, h, [] ]
|
110
122
|
end
|
@@ -112,6 +124,7 @@ class OMGF::HystericalRaisins
|
|
112
124
|
# GET /$DOMAIN/$KEY
|
113
125
|
def redirect_key(env, domain, key)
|
114
126
|
uris = mg_get_uris(domain, key, @get_paths_opts)
|
127
|
+
uris = @vp.verify(uris, 1, @verify_timeout).flatten!
|
115
128
|
|
116
129
|
return r(503, "") unless uris && dest = uris.shift
|
117
130
|
|
@@ -123,7 +136,6 @@ class OMGF::HystericalRaisins
|
|
123
136
|
}
|
124
137
|
|
125
138
|
unless reproxy?(env, key, h, location)
|
126
|
-
uris -= [ dest ]
|
127
139
|
uris.each_with_index { |uri,i| h["X-Alt-Location-#{i}"] = uri.to_s }
|
128
140
|
end
|
129
141
|
[ 302, h, [] ]
|
@@ -252,16 +264,39 @@ class OMGF::HystericalRaisins
|
|
252
264
|
return r(406, "key `#{key}' is not URI-friendly") if bad_key?(key)
|
253
265
|
return r(406, "key is too long") if key.size > 128
|
254
266
|
|
267
|
+
clen = env["CONTENT_LENGTH"]
|
268
|
+
|
255
269
|
# this was written before MogileFS supported empty files,
|
256
270
|
# but empty files waste DB space so we don't support them
|
257
271
|
# Not bothering with Transfer-Encoding: chunked, though...
|
258
|
-
return r(403, "empty files forbidden") if
|
272
|
+
return r(403, "empty files forbidden") if "0" == clen
|
259
273
|
|
260
274
|
params = query(env)
|
261
275
|
input = env["rack.input"]
|
262
276
|
paths = nil
|
263
277
|
retried = false
|
264
278
|
|
279
|
+
# prepare options for create_open/create_close:
|
280
|
+
new_file_opts = @new_file_opts.dup
|
281
|
+
|
282
|
+
# the original deployment of this created a class for every
|
283
|
+
# domain with the same class having the same name as the domain
|
284
|
+
new_file_opts[:class] = params['class'] || @default_class_cb[domain]
|
285
|
+
|
286
|
+
# try to give a Content-Length to the tracker
|
287
|
+
clen and new_file_opts[:content_length] = clen.to_i
|
288
|
+
|
289
|
+
if /\bContent-MD5\b/i =~ env["HTTP_TRAILER"]
|
290
|
+
# if the client will give the Content-MD5 as the trailer,
|
291
|
+
# we must lazily populate it since we're not guaranteed to
|
292
|
+
# have the trailer, yet (rack.input is lazily read on unicorn)
|
293
|
+
new_file_opts[:content_md5] = lambda { env["HTTP_CONTENT_MD5"] }
|
294
|
+
elsif cmd5 = env["HTTP_CONTENT_MD5"]
|
295
|
+
# maybe the client gave the Content-MD5 in the header
|
296
|
+
new_file_opts[:content_md5] = cmd5
|
297
|
+
end
|
298
|
+
|
299
|
+
buf = ""
|
265
300
|
begin
|
266
301
|
pool_use(domain) do |mg|
|
267
302
|
begin
|
@@ -281,14 +316,17 @@ class OMGF::HystericalRaisins
|
|
281
316
|
# good, not clobbering anything
|
282
317
|
end
|
283
318
|
|
284
|
-
#
|
285
|
-
|
286
|
-
|
287
|
-
|
319
|
+
# finally, upload the file
|
320
|
+
mg.new_file(key, new_file_opts) do |io|
|
321
|
+
while input.read(16384, buf)
|
322
|
+
io.write(buf)
|
323
|
+
end
|
324
|
+
end
|
288
325
|
end # pool_use
|
289
326
|
|
290
327
|
# should always return 201 if ! found, but we keep 200 for legacy
|
291
328
|
# compat if they're not logged in (via REMOTE_USER)
|
329
|
+
buf.clear
|
292
330
|
status = paths ? 204 : (env["REMOTE_USER"] ? 201 : 200)
|
293
331
|
r(status, "")
|
294
332
|
rescue MogileFS::Backend::UnregDomainError,
|
data/lib/omgf/pool.rb
CHANGED
@@ -40,9 +40,9 @@ module OMGF::Pool
|
|
40
40
|
pool_use(domain) { |mg| mg.get_uris(key, opts) }
|
41
41
|
end
|
42
42
|
|
43
|
-
def
|
43
|
+
def mg_size_and_uris(domain, key, opts) # :nodoc:
|
44
44
|
pool_use(domain) do |mg|
|
45
|
-
[ mg.size(key), mg.
|
45
|
+
[ mg.size(key), mg.get_uris(key, opts) ]
|
46
46
|
end
|
47
47
|
end
|
48
48
|
end
|
@@ -0,0 +1,241 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# :stopdoc:
|
3
|
+
# Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
|
4
|
+
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
5
|
+
# :startdoc:
|
6
|
+
require "thread"
|
7
|
+
require "omgf"
|
8
|
+
require "kcar"
|
9
|
+
require "kgio"
|
10
|
+
|
11
|
+
# This module makes HEAD requests with reusable HTTP connections to
|
12
|
+
# verify paths. This is faster than having the mogilefsd tracker
|
13
|
+
# verifying paths, and the client could have broken routing to some
|
14
|
+
# the storage nodes the mogilefsd tracker/monitor can see.
|
15
|
+
class OMGF::VerifyPaths
|
16
|
+
|
17
|
+
# private class
|
18
|
+
class HeadSock < Kgio::Socket # :nodoc:
|
19
|
+
VERSION = '1.0.0'
|
20
|
+
|
21
|
+
attr_reader :uri
|
22
|
+
attr_writer :retry_ok
|
23
|
+
|
24
|
+
def self.start(uri)
|
25
|
+
super(Socket.pack_sockaddr_in(uri.port, uri.host))
|
26
|
+
end
|
27
|
+
|
28
|
+
# returns true if the HTTP connection is reusable
|
29
|
+
def http_reusable?
|
30
|
+
rv = @kcar.keepalive?
|
31
|
+
@kcar.reset
|
32
|
+
@headers.clear
|
33
|
+
@buf.clear
|
34
|
+
@uri = nil
|
35
|
+
rv ? setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 1) : close
|
36
|
+
rv
|
37
|
+
end
|
38
|
+
|
39
|
+
def http_init(uri, retry_ok = true)
|
40
|
+
@uri = uri
|
41
|
+
unless defined?(@kcar)
|
42
|
+
@kcar = Kcar::Parser.new
|
43
|
+
@headers = {}
|
44
|
+
@buf = ""
|
45
|
+
end
|
46
|
+
@retry_ok = retry_ok
|
47
|
+
|
48
|
+
# prepare the HTTP request
|
49
|
+
@req = "HEAD #{@uri.request_uri} HTTP/1.1\r\n" \
|
50
|
+
"User-Agent: #{self.class}/#{VERSION}\r\n" \
|
51
|
+
"Host: #{@uri.host}:#{@uri.port}\r\n" \
|
52
|
+
"\r\n"
|
53
|
+
end
|
54
|
+
|
55
|
+
# returns an array result if successful
|
56
|
+
# returns :wait_readable or :wait_writable if incomplete
|
57
|
+
# returns a subclass of Exception on errors
|
58
|
+
# returns nil on premature EOF (twice)
|
59
|
+
def poll_iter(pollset)
|
60
|
+
case rv = kgio_trywrite(@req)
|
61
|
+
when nil, # done writing, start reading
|
62
|
+
String # partial write
|
63
|
+
@req = rv # continue looping
|
64
|
+
when Symbol
|
65
|
+
return pollset[self] = rv # busy
|
66
|
+
end while @req
|
67
|
+
|
68
|
+
case rv = kgio_tryread(666)
|
69
|
+
when String
|
70
|
+
if ary = @kcar.headers(@headers, @buf << rv)
|
71
|
+
pollset.delete(self)
|
72
|
+
return ary # success
|
73
|
+
end
|
74
|
+
# continue looping if incomplete
|
75
|
+
when Symbol
|
76
|
+
return pollset[self] = rv # busy
|
77
|
+
when nil # EOF (premature)
|
78
|
+
return maybe_retry(nil, pollset)
|
79
|
+
end while true
|
80
|
+
rescue => err
|
81
|
+
maybe_retry(err, pollset)
|
82
|
+
end
|
83
|
+
|
84
|
+
# returns the err object if we've already retried, nil otherwise
|
85
|
+
def maybe_retry(err, pollset)
|
86
|
+
pollset.delete(self)
|
87
|
+
return err unless @retry_ok
|
88
|
+
|
89
|
+
# always start a fresh connection on socket errors
|
90
|
+
sock = self.class.start(@uri)
|
91
|
+
sock.http_init(@uri, false)
|
92
|
+
pollset[sock] = :wait_writable
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def initialize(logger)
|
97
|
+
@pool = Hash.new { |hash,host_port| hash[host_port] = [] }
|
98
|
+
@logger = logger
|
99
|
+
@finishq = Queue.new
|
100
|
+
@finisher = nil
|
101
|
+
@lock = Mutex.new
|
102
|
+
@pid = $$
|
103
|
+
end
|
104
|
+
|
105
|
+
def error(msg)
|
106
|
+
@logger.error(msg) if @logger
|
107
|
+
end
|
108
|
+
|
109
|
+
def iter_check(ok, sock, pollset)
|
110
|
+
rv = sock.poll_iter(pollset)
|
111
|
+
case rv
|
112
|
+
when Symbol # in progress
|
113
|
+
when Array
|
114
|
+
code = rv[0].to_i
|
115
|
+
if 200 == code
|
116
|
+
ok << sock.uri
|
117
|
+
elsif code >= 100 && code <= 999
|
118
|
+
error("HEAD #{sock.uri} returned HTTP code: #{code}")
|
119
|
+
else
|
120
|
+
error("HEAD #{sock.uri} returned #{rv.inspect} (kcar bug?)")
|
121
|
+
end
|
122
|
+
sock_put(sock)
|
123
|
+
when nil # premature EOF
|
124
|
+
error("HEAD #{sock.uri} hit socket EOF")
|
125
|
+
else
|
126
|
+
# exception or some other error return value...
|
127
|
+
if rv.respond_to?(:message)
|
128
|
+
error("HEAD #{sock.uri} error: #{rv.message} (#{rv.class})")
|
129
|
+
else
|
130
|
+
error("HEAD #{sock.uri} error (#{rv.class}): #{rv.inspect}")
|
131
|
+
end
|
132
|
+
end
|
133
|
+
rv
|
134
|
+
end
|
135
|
+
|
136
|
+
# this runs in a background thread to cleanup all the requests
|
137
|
+
# that didn't finish quickly enough
|
138
|
+
def finisher(timeout = 10000)
|
139
|
+
begin
|
140
|
+
pollset = @finishq.pop # park here when idle
|
141
|
+
|
142
|
+
while ready = Kgio.poll(pollset.dup, timeout)
|
143
|
+
ready.each_key do |sock|
|
144
|
+
sock.retry_ok = false
|
145
|
+
# try to return good sockets back to the pool
|
146
|
+
iter_check([], sock, pollset)
|
147
|
+
end
|
148
|
+
|
149
|
+
# try to stuff the pollset as much as possible for further looping
|
150
|
+
while more = (@finishq.pop(true) rescue nil)
|
151
|
+
pollset.merge!(more)
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
# connections timed out, kill them
|
156
|
+
pollset.each_key { |sock| sock.close }
|
157
|
+
rescue => err
|
158
|
+
error("#{err.message} (#{err.class})")
|
159
|
+
end while true
|
160
|
+
end
|
161
|
+
|
162
|
+
# reorders URIs based on response time
|
163
|
+
# This is the main method of this class
|
164
|
+
def verify(uris, count, timeout)
|
165
|
+
tout = (timeout * 1000).to_i
|
166
|
+
pollset = {}
|
167
|
+
ok = []
|
168
|
+
|
169
|
+
uris.each do |uri|
|
170
|
+
sock = sock_get(uri) and iter_check(ok, sock, pollset)
|
171
|
+
end
|
172
|
+
|
173
|
+
while ok.size < count && tout > 0 && ! pollset.empty?
|
174
|
+
t0 = Time.now
|
175
|
+
ready = Kgio.poll(pollset.dup, tout) or break
|
176
|
+
tout -= ((Time.now - t0) * 1000).to_i
|
177
|
+
|
178
|
+
ready.each_key do |sock|
|
179
|
+
iter_check(ok, sock, pollset)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
finish(pollset) unless pollset.empty?
|
184
|
+
[ok, uris - ok] # good URLs first
|
185
|
+
end
|
186
|
+
|
187
|
+
# recover any unfinished URLs in pollset asynchronously in the background
|
188
|
+
def finish(pollset) # :nodoc:
|
189
|
+
@finishq.push(pollset)
|
190
|
+
@lock.synchronize do
|
191
|
+
unless @finisher && @finisher.alive?
|
192
|
+
@finisher = Thread.new { finisher }
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
# returns a string key for the connection pool
|
198
|
+
def key_for(uri)
|
199
|
+
"#{uri.host}:#{uri.port}"
|
200
|
+
end
|
201
|
+
|
202
|
+
# initializes a cached connection for +uri+ or creates a new one
|
203
|
+
def sock_get(uri)
|
204
|
+
key = key_for(uri)
|
205
|
+
|
206
|
+
# detect forks and prevent sharing of connected sockets across processes
|
207
|
+
@lock.synchronize do
|
208
|
+
if @pid != $$
|
209
|
+
@pid = $$
|
210
|
+
@pool.clear
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
while sock = @lock.synchronize { @pool[key].pop }
|
215
|
+
begin
|
216
|
+
# check if sock is still alive and idle
|
217
|
+
# :wait_readable is good here
|
218
|
+
break if sock.kgio_tryread(1) == :wait_readable
|
219
|
+
rescue
|
220
|
+
# ignore socket errors, we'll just give them a new socket
|
221
|
+
# socket should've been idle, but it was not (or EOFed on us)
|
222
|
+
# give them a new one
|
223
|
+
end
|
224
|
+
sock.close
|
225
|
+
end
|
226
|
+
|
227
|
+
sock ||= HeadSock.start(uri)
|
228
|
+
sock.http_init(uri)
|
229
|
+
sock
|
230
|
+
rescue
|
231
|
+
# we'll return nil on any errors
|
232
|
+
end
|
233
|
+
|
234
|
+
# returns an idle socket to the pool
|
235
|
+
def sock_put(sock)
|
236
|
+
key = key_for(sock.uri)
|
237
|
+
sock.http_reusable? and @lock.synchronize { @pool[key] << sock }
|
238
|
+
rescue => err
|
239
|
+
error("HTTP reuse check failed: #{err.message} (#{err.class})")
|
240
|
+
end
|
241
|
+
end
|
data/omgf.gemspec
CHANGED
@@ -20,6 +20,8 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.rdoc_options = rdoc_options
|
21
21
|
s.test_files = Dir['test/test_*.rb']
|
22
22
|
s.add_dependency('rack', ['~> 1.3'])
|
23
|
+
s.add_dependency('kgio', ['~> 2.7'])
|
24
|
+
s.add_dependency('kcar', ['~> 0.3'])
|
23
25
|
s.add_dependency('mogilefs-client', ['~> 3.1'])
|
24
26
|
|
25
27
|
s.licenses = %w(AGPLv3+)
|
data/test/integration.rb
CHANGED
@@ -67,9 +67,9 @@ module TestMogileFSIntegration
|
|
67
67
|
[ status, out, err ]
|
68
68
|
end
|
69
69
|
|
70
|
-
def setup_mogilefs(plugins = nil)
|
70
|
+
def setup_mogilefs(plugins = nil, mogstored = "mogstored")
|
71
71
|
@test_host = "127.0.0.1"
|
72
|
-
setup_mogstored
|
72
|
+
setup_mogstored(mogstored)
|
73
73
|
@tracker = TCPServer.new(@test_host, 0)
|
74
74
|
@tracker_port = @tracker.addr[1]
|
75
75
|
|
@@ -173,7 +173,7 @@ EOF
|
|
173
173
|
raise "#{uri} failed to appear: #{res.inspect}"
|
174
174
|
end
|
175
175
|
|
176
|
-
def setup_mogstored
|
176
|
+
def setup_mogstored(mogstored = "mogstored")
|
177
177
|
@docroot = Dir.mktmpdir(["mogfresh", "docroot"])
|
178
178
|
Dir.mkdir("#@docroot/dev1")
|
179
179
|
Dir.mkdir("#@docroot/dev2")
|
@@ -194,7 +194,7 @@ EOF
|
|
194
194
|
@mogstored_mgmt.close
|
195
195
|
@mogstored_http.close
|
196
196
|
|
197
|
-
x!(
|
197
|
+
x!(mogstored, "--daemon", "--config=#{@mogstored_conf.path}")
|
198
198
|
wait_for_port @mogstored_mgmt_port
|
199
199
|
wait_for_port @mogstored_http_port
|
200
200
|
end
|
@@ -4,15 +4,17 @@ require './test/integration'
|
|
4
4
|
require 'rack/mock'
|
5
5
|
require 'open-uri'
|
6
6
|
require 'omgf/hysterical_raisins'
|
7
|
+
require 'digest/md5'
|
7
8
|
|
8
9
|
class TestHystericalRaisins < Test::Unit::TestCase
|
9
10
|
include TestMogileFSIntegration
|
10
11
|
def setup
|
11
12
|
setup_mogilefs
|
12
|
-
@app = OMGF::HystericalRaisins.new(:hosts => @hosts)
|
13
|
-
@req = Rack::MockRequest.new(@app)
|
14
13
|
@err = StringIO.new
|
15
|
-
|
14
|
+
logger = Logger.new(@err)
|
15
|
+
@opts = { "rack.logger" => logger }
|
16
|
+
@app = OMGF::HystericalRaisins.new(:hosts => @hosts, :logger => logger)
|
17
|
+
@req = Rack::MockRequest.new(@app)
|
16
18
|
@admin.create_domain("testdom")
|
17
19
|
end
|
18
20
|
|
@@ -186,13 +188,21 @@ class TestHystericalRaisins < Test::Unit::TestCase
|
|
186
188
|
|
187
189
|
# wait for replication
|
188
190
|
if ENV["EXPENSIVE"]
|
189
|
-
|
190
|
-
resp["X-URL-1"] and break
|
191
|
-
sleep 0.1
|
191
|
+
50.times do
|
192
192
|
resp = @req.head("/testdom/key", @opts)
|
193
|
+
assert_equal 200, resp.status, resp.inspect
|
194
|
+
resp["X-URL-1"] and break
|
195
|
+
sleep 0.5
|
193
196
|
end
|
194
|
-
assert_kind_of String, resp["X-URL-1"]
|
197
|
+
assert_kind_of String, resp["X-URL-1"], resp.inspect
|
195
198
|
assert_equal "BLAH", open(resp["X-URL-1"]).read
|
199
|
+
|
200
|
+
# Location should not be in X-Alt-Location, too
|
201
|
+
resp = @req.get("/testdom/key", @opts)
|
202
|
+
assert_equal 302, resp.status, resp.inspect
|
203
|
+
assert_kind_of URI, URI(resp["Location"])
|
204
|
+
assert_kind_of URI, URI(resp["X-Alt-Location-0"])
|
205
|
+
assert_nil resp["X-Alt-Location-1"]
|
196
206
|
end
|
197
207
|
|
198
208
|
reproxy_test
|
@@ -232,6 +242,24 @@ class TestHystericalRaisins < Test::Unit::TestCase
|
|
232
242
|
@app.instance_variable_set(:@reproxy_path, nil)
|
233
243
|
end
|
234
244
|
|
245
|
+
def test_checksums
|
246
|
+
mogadm!("class", "add", "testdom", "md5", "--hashtype=MD5")
|
247
|
+
100.times do
|
248
|
+
@admin.get_domains["testdom"]["md5"] and break
|
249
|
+
sleep 0.5
|
250
|
+
end
|
251
|
+
assert_equal "MD5", @admin.get_domains["testdom"]["md5"]["hashtype"]
|
252
|
+
sio = StringIO.new("HELLO")
|
253
|
+
md5 = [ Digest::MD5.digest(sio.string) ].pack('m0')
|
254
|
+
opts = @opts.merge(input: sio, method: "PUT", "HTTP_CONTENT_MD5" => md5)
|
255
|
+
resp = @req.put("/testdom/cc?class=md5", opts)
|
256
|
+
assert_equal 200, resp.status
|
257
|
+
@mg = MogileFS::MogileFS.new(hosts: @hosts, domain: "testdom")
|
258
|
+
info = @mg.file_info("cc")
|
259
|
+
assert_equal "md5", info["class"]
|
260
|
+
assert_equal "MD5:#{Digest::MD5.hexdigest(sio.string)}", info["checksum"]
|
261
|
+
end
|
262
|
+
|
235
263
|
def teardown
|
236
264
|
teardown_mogilefs
|
237
265
|
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
|
2
|
+
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
3
|
+
require './test/integration'
|
4
|
+
require 'rack/mock'
|
5
|
+
require 'open-uri'
|
6
|
+
require 'omgf/hysterical_raisins'
|
7
|
+
require 'digest/md5'
|
8
|
+
|
9
|
+
class TestHystericalRaisinsCmogstored < Test::Unit::TestCase
|
10
|
+
include TestMogileFSIntegration
|
11
|
+
def setup
|
12
|
+
setup_mogilefs(nil, "cmogstored")
|
13
|
+
@err = StringIO.new
|
14
|
+
logger = Logger.new(@err)
|
15
|
+
@opts = { "rack.logger" => logger }
|
16
|
+
@app = OMGF::HystericalRaisins.new(:hosts => @hosts, :logger => logger)
|
17
|
+
@req = Rack::MockRequest.new(@app)
|
18
|
+
@admin.create_domain("testdom")
|
19
|
+
end
|
20
|
+
|
21
|
+
def test_checksums_trailer
|
22
|
+
mogadm!("class", "add", "testdom", "md5", "--hashtype=MD5")
|
23
|
+
100.times do
|
24
|
+
@admin.get_domains["testdom"]["md5"] and break
|
25
|
+
sleep 0.5
|
26
|
+
end
|
27
|
+
assert_equal "MD5", @admin.get_domains["testdom"]["md5"]["hashtype"]
|
28
|
+
|
29
|
+
sio = StringIO.new("HELLO")
|
30
|
+
def sio.read(*_)
|
31
|
+
rv = super
|
32
|
+
if rv == nil
|
33
|
+
md5 = [ Digest::MD5.digest(string) ].pack('m0')
|
34
|
+
@rack_env["HTTP_CONTENT_MD5"] = md5
|
35
|
+
end
|
36
|
+
rv
|
37
|
+
end
|
38
|
+
|
39
|
+
opts = @opts.merge(input: sio, method: "PUT")
|
40
|
+
opts["HTTP_TRAILER"] = "Content-MD5"
|
41
|
+
env = @req.class.env_for("/testdom/cc?class=md5", opts)
|
42
|
+
sio.instance_variable_set(:@rack_env, env)
|
43
|
+
status, _, _ = @app.call(env)
|
44
|
+
assert_equal 200, status, @err.string
|
45
|
+
@mg = MogileFS::MogileFS.new(hosts: @hosts, domain: "testdom")
|
46
|
+
info = @mg.file_info("cc")
|
47
|
+
assert_equal "md5", info["class"]
|
48
|
+
assert_equal "MD5:#{Digest::MD5.hexdigest(sio.string)}", info["checksum"]
|
49
|
+
end
|
50
|
+
|
51
|
+
def teardown
|
52
|
+
teardown_mogilefs
|
53
|
+
end
|
54
|
+
end if `which cmogstored 2>/dev/null` =~ /\bcmogstored\b/
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: !binary |-
|
3
3
|
b21nZg==
|
4
4
|
version: !ruby/object:Gem::Version
|
5
|
-
version: 0.0.
|
5
|
+
version: 0.0.1.GIT
|
6
6
|
prerelease: 6
|
7
7
|
platform: ruby
|
8
8
|
authors:
|
@@ -11,7 +11,7 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date: 2012-06-
|
14
|
+
date: 2012-06-23 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: !binary |-
|
@@ -34,6 +34,48 @@ dependencies:
|
|
34
34
|
- !ruby/object:Gem::Version
|
35
35
|
version: !binary |-
|
36
36
|
MS4z
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: !binary |-
|
39
|
+
a2dpbw==
|
40
|
+
requirement: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - !binary |-
|
44
|
+
fj4=
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: !binary |-
|
47
|
+
Mi43
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
none: false
|
52
|
+
requirements:
|
53
|
+
- - !binary |-
|
54
|
+
fj4=
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: !binary |-
|
57
|
+
Mi43
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: !binary |-
|
60
|
+
a2Nhcg==
|
61
|
+
requirement: !ruby/object:Gem::Requirement
|
62
|
+
none: false
|
63
|
+
requirements:
|
64
|
+
- - !binary |-
|
65
|
+
fj4=
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: !binary |-
|
68
|
+
MC4z
|
69
|
+
type: :runtime
|
70
|
+
prerelease: false
|
71
|
+
version_requirements: !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - !binary |-
|
75
|
+
fj4=
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: !binary |-
|
78
|
+
MC4z
|
37
79
|
- !ruby/object:Gem::Dependency
|
38
80
|
name: !binary |-
|
39
81
|
bW9naWxlZnMtY2xpZW50
|
@@ -71,6 +113,8 @@ extra_rdoc_files:
|
|
71
113
|
- lib/omgf.rb
|
72
114
|
- lib/omgf/hysterical_raisins.rb
|
73
115
|
- lib/omgf/pool.rb
|
116
|
+
- lib/omgf/verify_paths.rb
|
117
|
+
- lib/omgf/version.rb
|
74
118
|
- NEWS
|
75
119
|
- README
|
76
120
|
files:
|
@@ -88,14 +132,18 @@ files:
|
|
88
132
|
- README
|
89
133
|
- examples/hyst.README
|
90
134
|
- examples/hyst.bash
|
135
|
+
- examples/verify_paths_manual_test.rb
|
91
136
|
- lib/omgf.rb
|
92
137
|
- lib/omgf/hysterical_raisins.rb
|
93
138
|
- lib/omgf/pool.rb
|
139
|
+
- lib/omgf/verify_paths.rb
|
140
|
+
- lib/omgf/version.rb
|
94
141
|
- omgf.gemspec
|
95
142
|
- pkg.mk
|
96
143
|
- test/integration.rb
|
97
144
|
- test/test_hyst.rb
|
98
145
|
- test/test_hysterical_raisins.rb
|
146
|
+
- test/test_hysterical_raisins_cmogstored.rb
|
99
147
|
homepage: http://bogomips.org/omgf/
|
100
148
|
licenses:
|
101
149
|
- !binary |-
|
@@ -127,6 +175,8 @@ signing_key:
|
|
127
175
|
specification_version: 3
|
128
176
|
summary: hysterical REST API for MogileFS using Rack
|
129
177
|
test_files:
|
178
|
+
- !binary |-
|
179
|
+
dGVzdC90ZXN0X2h5c3RlcmljYWxfcmFpc2luc19jbW9nc3RvcmVkLnJi
|
130
180
|
- !binary |-
|
131
181
|
dGVzdC90ZXN0X2h5c3QucmI=
|
132
182
|
- !binary |-
|