omgf 0.0.0.GIT → 0.0.1.GIT
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/GIT-VERSION-GEN +26 -36
- data/GNUmakefile +1 -0
- data/examples/verify_paths_manual_test.rb +20 -0
- data/lib/omgf.rb +1 -0
- data/lib/omgf/hysterical_raisins.rb +48 -10
- data/lib/omgf/pool.rb +2 -2
- data/lib/omgf/verify_paths.rb +241 -0
- data/omgf.gemspec +2 -0
- data/test/integration.rb +4 -4
- data/test/test_hysterical_raisins.rb +35 -7
- data/test/test_hysterical_raisins_cmogstored.rb +54 -0
- metadata +52 -2
data/.gitignore
CHANGED
data/GIT-VERSION-GEN
CHANGED
@@ -1,40 +1,30 @@
|
|
1
|
-
#!/bin/
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
'
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
CONSTANT = "OMGF::VERSION"
|
3
|
+
RVF = "lib/omgf/version.rb"
|
4
|
+
GVF = "GIT-VERSION-FILE"
|
5
|
+
DEF_VER = "v0.0.1.GIT"
|
6
|
+
vn = DEF_VER
|
8
7
|
|
9
8
|
# First see if there is a version file (included in release tarballs),
|
10
9
|
# then try git-describe, then default.
|
11
|
-
if
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
then
|
24
|
-
VN=$(echo "$VN" | sed -e 's/-/./g');
|
25
|
-
else
|
26
|
-
VN="$DEF_VER"
|
27
|
-
fi
|
28
|
-
|
29
|
-
VN=$(expr "$VN" : v*'\(.*\)')
|
10
|
+
if File.exist?(".git")
|
11
|
+
describe = `git describe --abbrev=4 HEAD 2>/dev/null`.strip
|
12
|
+
case describe
|
13
|
+
when /\Av[0-9]*/
|
14
|
+
vn = describe
|
15
|
+
system(*%w(git update-index -q --refresh))
|
16
|
+
unless `git diff-index --name-only HEAD --`.chomp.empty?
|
17
|
+
vn << "-dirty"
|
18
|
+
end
|
19
|
+
vn.tr!('-', '.')
|
20
|
+
end
|
21
|
+
end
|
30
22
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
echo "GIT_VERSION = $VN" >$GVF
|
40
|
-
}
|
23
|
+
vn = vn.sub!(/\Av/, "")
|
24
|
+
new_ruby_version = "#{CONSTANT} = '#{vn}'\n"
|
25
|
+
cur_ruby_version = File.read(RVF) rescue nil
|
26
|
+
if new_ruby_version != cur_ruby_version
|
27
|
+
File.open(GVF, "w") { |fp| fp.write("GIT_VERSION = #{vn}\n") }
|
28
|
+
File.open(RVF, "w") { |fp| fp.write(new_ruby_version) }
|
29
|
+
end
|
30
|
+
puts vn if $0 == __FILE__
|
data/GNUmakefile
CHANGED
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
require "omgf/verify_paths"
|
3
|
+
require "logger"
|
4
|
+
require "uri"
|
5
|
+
require "pp"
|
6
|
+
vp = OMGF::VerifyPaths.new(Logger.new($stderr))
|
7
|
+
urls = %w(
|
8
|
+
http://yhbt.net/
|
9
|
+
http://bogomips.org/omgf.git
|
10
|
+
http://bogomips.org/ozZZZZmgf.git
|
11
|
+
http://bogomips.org/ozZZZZ
|
12
|
+
http://bogomips.org/
|
13
|
+
http://127.0.0.1:666/
|
14
|
+
)
|
15
|
+
uris = urls.map { |uri| URI(uri) }
|
16
|
+
|
17
|
+
pp vp.verify(uris, 1, 0.1)
|
18
|
+
uris = urls.map { |uri| URI(uri) }
|
19
|
+
sleep 1
|
20
|
+
pp vp.verify(uris, 1, 0.1)
|
data/lib/omgf.rb
CHANGED
@@ -4,6 +4,7 @@
|
|
4
4
|
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
5
5
|
# :startdoc:
|
6
6
|
require "omgf/pool"
|
7
|
+
require "omgf/verify_paths"
|
7
8
|
require "time"
|
8
9
|
require "json"
|
9
10
|
require "mogilefs"
|
@@ -41,7 +42,15 @@ class OMGF::HystericalRaisins
|
|
41
42
|
noverify: opts[:noverify],
|
42
43
|
pathcount: opts[:pathcount] || 0x7fffffff,
|
43
44
|
}
|
45
|
+
@new_file_opts = {
|
46
|
+
content_md5: opts[:content_md5], # :trailer is acceptable here
|
47
|
+
|
48
|
+
# largefile: auto-selects based on env["CONTENT_LENGTH"]
|
49
|
+
largefile: opts[:largefile] || :stream,
|
50
|
+
}
|
44
51
|
@put_overwrite_header = opts[:put_overwrite_header] || "HTTP_X_OMGF_FORCE"
|
52
|
+
@vp = OMGF::VerifyPaths.new(opts[:logger])
|
53
|
+
@verify_timeout = opts[:verify_timeout] || 0.5
|
45
54
|
pool_init(mg_opts)
|
46
55
|
end
|
47
56
|
|
@@ -98,13 +107,16 @@ class OMGF::HystericalRaisins
|
|
98
107
|
|
99
108
|
# HEAD /$DOMAIN/$KEY
|
100
109
|
def stat_key(env, domain, key)
|
101
|
-
size,
|
102
|
-
|
110
|
+
size, uris = mg_size_and_uris(domain, key, @get_paths_opts)
|
111
|
+
uris = @vp.verify(uris, 1, @verify_timeout).flatten!
|
112
|
+
|
113
|
+
return r(503, "") unless uris && uris[0]
|
114
|
+
|
103
115
|
h = { "Content-Length" => size.to_s }
|
104
116
|
fn = filename(h, query(env)) || key
|
105
117
|
h["Content-Type"] = key_mime_type(fn)
|
106
|
-
unless reproxy?(env, key, h,
|
107
|
-
|
118
|
+
unless reproxy?(env, key, h, uris[0].to_s)
|
119
|
+
uris.each_with_index { |uri,i| h["X-URL-#{i}"] = uri.to_s }
|
108
120
|
end
|
109
121
|
[ 200, h, [] ]
|
110
122
|
end
|
@@ -112,6 +124,7 @@ class OMGF::HystericalRaisins
|
|
112
124
|
# GET /$DOMAIN/$KEY
|
113
125
|
def redirect_key(env, domain, key)
|
114
126
|
uris = mg_get_uris(domain, key, @get_paths_opts)
|
127
|
+
uris = @vp.verify(uris, 1, @verify_timeout).flatten!
|
115
128
|
|
116
129
|
return r(503, "") unless uris && dest = uris.shift
|
117
130
|
|
@@ -123,7 +136,6 @@ class OMGF::HystericalRaisins
|
|
123
136
|
}
|
124
137
|
|
125
138
|
unless reproxy?(env, key, h, location)
|
126
|
-
uris -= [ dest ]
|
127
139
|
uris.each_with_index { |uri,i| h["X-Alt-Location-#{i}"] = uri.to_s }
|
128
140
|
end
|
129
141
|
[ 302, h, [] ]
|
@@ -252,16 +264,39 @@ class OMGF::HystericalRaisins
|
|
252
264
|
return r(406, "key `#{key}' is not URI-friendly") if bad_key?(key)
|
253
265
|
return r(406, "key is too long") if key.size > 128
|
254
266
|
|
267
|
+
clen = env["CONTENT_LENGTH"]
|
268
|
+
|
255
269
|
# this was written before MogileFS supported empty files,
|
256
270
|
# but empty files waste DB space so we don't support them
|
257
271
|
# Not bothering with Transfer-Encoding: chunked, though...
|
258
|
-
return r(403, "empty files forbidden") if
|
272
|
+
return r(403, "empty files forbidden") if "0" == clen
|
259
273
|
|
260
274
|
params = query(env)
|
261
275
|
input = env["rack.input"]
|
262
276
|
paths = nil
|
263
277
|
retried = false
|
264
278
|
|
279
|
+
# prepare options for create_open/create_close:
|
280
|
+
new_file_opts = @new_file_opts.dup
|
281
|
+
|
282
|
+
# the original deployment of this created a class for every
|
283
|
+
# domain with the same class having the same name as the domain
|
284
|
+
new_file_opts[:class] = params['class'] || @default_class_cb[domain]
|
285
|
+
|
286
|
+
# try to give a Content-Length to the tracker
|
287
|
+
clen and new_file_opts[:content_length] = clen.to_i
|
288
|
+
|
289
|
+
if /\bContent-MD5\b/i =~ env["HTTP_TRAILER"]
|
290
|
+
# if the client will give the Content-MD5 as the trailer,
|
291
|
+
# we must lazily populate it since we're not guaranteed to
|
292
|
+
# have the trailer, yet (rack.input is lazily read on unicorn)
|
293
|
+
new_file_opts[:content_md5] = lambda { env["HTTP_CONTENT_MD5"] }
|
294
|
+
elsif cmd5 = env["HTTP_CONTENT_MD5"]
|
295
|
+
# maybe the client gave the Content-MD5 in the header
|
296
|
+
new_file_opts[:content_md5] = cmd5
|
297
|
+
end
|
298
|
+
|
299
|
+
buf = ""
|
265
300
|
begin
|
266
301
|
pool_use(domain) do |mg|
|
267
302
|
begin
|
@@ -281,14 +316,17 @@ class OMGF::HystericalRaisins
|
|
281
316
|
# good, not clobbering anything
|
282
317
|
end
|
283
318
|
|
284
|
-
#
|
285
|
-
|
286
|
-
|
287
|
-
|
319
|
+
# finally, upload the file
|
320
|
+
mg.new_file(key, new_file_opts) do |io|
|
321
|
+
while input.read(16384, buf)
|
322
|
+
io.write(buf)
|
323
|
+
end
|
324
|
+
end
|
288
325
|
end # pool_use
|
289
326
|
|
290
327
|
# should always return 201 if ! found, but we keep 200 for legacy
|
291
328
|
# compat if they're not logged in (via REMOTE_USER)
|
329
|
+
buf.clear
|
292
330
|
status = paths ? 204 : (env["REMOTE_USER"] ? 201 : 200)
|
293
331
|
r(status, "")
|
294
332
|
rescue MogileFS::Backend::UnregDomainError,
|
data/lib/omgf/pool.rb
CHANGED
@@ -40,9 +40,9 @@ module OMGF::Pool
|
|
40
40
|
pool_use(domain) { |mg| mg.get_uris(key, opts) }
|
41
41
|
end
|
42
42
|
|
43
|
-
def
|
43
|
+
def mg_size_and_uris(domain, key, opts) # :nodoc:
|
44
44
|
pool_use(domain) do |mg|
|
45
|
-
[ mg.size(key), mg.
|
45
|
+
[ mg.size(key), mg.get_uris(key, opts) ]
|
46
46
|
end
|
47
47
|
end
|
48
48
|
end
|
@@ -0,0 +1,241 @@
|
|
1
|
+
# -*- encoding: binary -*-
|
2
|
+
# :stopdoc:
|
3
|
+
# Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
|
4
|
+
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
5
|
+
# :startdoc:
|
6
|
+
require "thread"
|
7
|
+
require "omgf"
|
8
|
+
require "kcar"
|
9
|
+
require "kgio"
|
10
|
+
|
11
|
+
# This module makes HEAD requests with reusable HTTP connections to
|
12
|
+
# verify paths. This is faster than having the mogilefsd tracker
|
13
|
+
# verifying paths, and the client could have broken routing to some
|
14
|
+
# the storage nodes the mogilefsd tracker/monitor can see.
|
15
|
+
class OMGF::VerifyPaths
|
16
|
+
|
17
|
+
# private class
|
18
|
+
class HeadSock < Kgio::Socket # :nodoc:
|
19
|
+
VERSION = '1.0.0'
|
20
|
+
|
21
|
+
attr_reader :uri
|
22
|
+
attr_writer :retry_ok
|
23
|
+
|
24
|
+
def self.start(uri)
|
25
|
+
super(Socket.pack_sockaddr_in(uri.port, uri.host))
|
26
|
+
end
|
27
|
+
|
28
|
+
# returns true if the HTTP connection is reusable
|
29
|
+
def http_reusable?
|
30
|
+
rv = @kcar.keepalive?
|
31
|
+
@kcar.reset
|
32
|
+
@headers.clear
|
33
|
+
@buf.clear
|
34
|
+
@uri = nil
|
35
|
+
rv ? setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 1) : close
|
36
|
+
rv
|
37
|
+
end
|
38
|
+
|
39
|
+
def http_init(uri, retry_ok = true)
|
40
|
+
@uri = uri
|
41
|
+
unless defined?(@kcar)
|
42
|
+
@kcar = Kcar::Parser.new
|
43
|
+
@headers = {}
|
44
|
+
@buf = ""
|
45
|
+
end
|
46
|
+
@retry_ok = retry_ok
|
47
|
+
|
48
|
+
# prepare the HTTP request
|
49
|
+
@req = "HEAD #{@uri.request_uri} HTTP/1.1\r\n" \
|
50
|
+
"User-Agent: #{self.class}/#{VERSION}\r\n" \
|
51
|
+
"Host: #{@uri.host}:#{@uri.port}\r\n" \
|
52
|
+
"\r\n"
|
53
|
+
end
|
54
|
+
|
55
|
+
# returns an array result if successful
|
56
|
+
# returns :wait_readable or :wait_writable if incomplete
|
57
|
+
# returns a subclass of Exception on errors
|
58
|
+
# returns nil on premature EOF (twice)
|
59
|
+
def poll_iter(pollset)
|
60
|
+
case rv = kgio_trywrite(@req)
|
61
|
+
when nil, # done writing, start reading
|
62
|
+
String # partial write
|
63
|
+
@req = rv # continue looping
|
64
|
+
when Symbol
|
65
|
+
return pollset[self] = rv # busy
|
66
|
+
end while @req
|
67
|
+
|
68
|
+
case rv = kgio_tryread(666)
|
69
|
+
when String
|
70
|
+
if ary = @kcar.headers(@headers, @buf << rv)
|
71
|
+
pollset.delete(self)
|
72
|
+
return ary # success
|
73
|
+
end
|
74
|
+
# continue looping if incomplete
|
75
|
+
when Symbol
|
76
|
+
return pollset[self] = rv # busy
|
77
|
+
when nil # EOF (premature)
|
78
|
+
return maybe_retry(nil, pollset)
|
79
|
+
end while true
|
80
|
+
rescue => err
|
81
|
+
maybe_retry(err, pollset)
|
82
|
+
end
|
83
|
+
|
84
|
+
# returns the err object if we've already retried, nil otherwise
|
85
|
+
def maybe_retry(err, pollset)
|
86
|
+
pollset.delete(self)
|
87
|
+
return err unless @retry_ok
|
88
|
+
|
89
|
+
# always start a fresh connection on socket errors
|
90
|
+
sock = self.class.start(@uri)
|
91
|
+
sock.http_init(@uri, false)
|
92
|
+
pollset[sock] = :wait_writable
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def initialize(logger)
|
97
|
+
@pool = Hash.new { |hash,host_port| hash[host_port] = [] }
|
98
|
+
@logger = logger
|
99
|
+
@finishq = Queue.new
|
100
|
+
@finisher = nil
|
101
|
+
@lock = Mutex.new
|
102
|
+
@pid = $$
|
103
|
+
end
|
104
|
+
|
105
|
+
def error(msg)
|
106
|
+
@logger.error(msg) if @logger
|
107
|
+
end
|
108
|
+
|
109
|
+
def iter_check(ok, sock, pollset)
|
110
|
+
rv = sock.poll_iter(pollset)
|
111
|
+
case rv
|
112
|
+
when Symbol # in progress
|
113
|
+
when Array
|
114
|
+
code = rv[0].to_i
|
115
|
+
if 200 == code
|
116
|
+
ok << sock.uri
|
117
|
+
elsif code >= 100 && code <= 999
|
118
|
+
error("HEAD #{sock.uri} returned HTTP code: #{code}")
|
119
|
+
else
|
120
|
+
error("HEAD #{sock.uri} returned #{rv.inspect} (kcar bug?)")
|
121
|
+
end
|
122
|
+
sock_put(sock)
|
123
|
+
when nil # premature EOF
|
124
|
+
error("HEAD #{sock.uri} hit socket EOF")
|
125
|
+
else
|
126
|
+
# exception or some other error return value...
|
127
|
+
if rv.respond_to?(:message)
|
128
|
+
error("HEAD #{sock.uri} error: #{rv.message} (#{rv.class})")
|
129
|
+
else
|
130
|
+
error("HEAD #{sock.uri} error (#{rv.class}): #{rv.inspect}")
|
131
|
+
end
|
132
|
+
end
|
133
|
+
rv
|
134
|
+
end
|
135
|
+
|
136
|
+
# this runs in a background thread to cleanup all the requests
|
137
|
+
# that didn't finish quickly enough
|
138
|
+
def finisher(timeout = 10000)
|
139
|
+
begin
|
140
|
+
pollset = @finishq.pop # park here when idle
|
141
|
+
|
142
|
+
while ready = Kgio.poll(pollset.dup, timeout)
|
143
|
+
ready.each_key do |sock|
|
144
|
+
sock.retry_ok = false
|
145
|
+
# try to return good sockets back to the pool
|
146
|
+
iter_check([], sock, pollset)
|
147
|
+
end
|
148
|
+
|
149
|
+
# try to stuff the pollset as much as possible for further looping
|
150
|
+
while more = (@finishq.pop(true) rescue nil)
|
151
|
+
pollset.merge!(more)
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
# connections timed out, kill them
|
156
|
+
pollset.each_key { |sock| sock.close }
|
157
|
+
rescue => err
|
158
|
+
error("#{err.message} (#{err.class})")
|
159
|
+
end while true
|
160
|
+
end
|
161
|
+
|
162
|
+
# reorders URIs based on response time
|
163
|
+
# This is the main method of this class
|
164
|
+
def verify(uris, count, timeout)
|
165
|
+
tout = (timeout * 1000).to_i
|
166
|
+
pollset = {}
|
167
|
+
ok = []
|
168
|
+
|
169
|
+
uris.each do |uri|
|
170
|
+
sock = sock_get(uri) and iter_check(ok, sock, pollset)
|
171
|
+
end
|
172
|
+
|
173
|
+
while ok.size < count && tout > 0 && ! pollset.empty?
|
174
|
+
t0 = Time.now
|
175
|
+
ready = Kgio.poll(pollset.dup, tout) or break
|
176
|
+
tout -= ((Time.now - t0) * 1000).to_i
|
177
|
+
|
178
|
+
ready.each_key do |sock|
|
179
|
+
iter_check(ok, sock, pollset)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
finish(pollset) unless pollset.empty?
|
184
|
+
[ok, uris - ok] # good URLs first
|
185
|
+
end
|
186
|
+
|
187
|
+
# recover any unfinished URLs in pollset asynchronously in the background
|
188
|
+
def finish(pollset) # :nodoc:
|
189
|
+
@finishq.push(pollset)
|
190
|
+
@lock.synchronize do
|
191
|
+
unless @finisher && @finisher.alive?
|
192
|
+
@finisher = Thread.new { finisher }
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
# returns a string key for the connection pool
|
198
|
+
def key_for(uri)
|
199
|
+
"#{uri.host}:#{uri.port}"
|
200
|
+
end
|
201
|
+
|
202
|
+
# initializes a cached connection for +uri+ or creates a new one
|
203
|
+
def sock_get(uri)
|
204
|
+
key = key_for(uri)
|
205
|
+
|
206
|
+
# detect forks and prevent sharing of connected sockets across processes
|
207
|
+
@lock.synchronize do
|
208
|
+
if @pid != $$
|
209
|
+
@pid = $$
|
210
|
+
@pool.clear
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
while sock = @lock.synchronize { @pool[key].pop }
|
215
|
+
begin
|
216
|
+
# check if sock is still alive and idle
|
217
|
+
# :wait_readable is good here
|
218
|
+
break if sock.kgio_tryread(1) == :wait_readable
|
219
|
+
rescue
|
220
|
+
# ignore socket errors, we'll just give them a new socket
|
221
|
+
# socket should've been idle, but it was not (or EOFed on us)
|
222
|
+
# give them a new one
|
223
|
+
end
|
224
|
+
sock.close
|
225
|
+
end
|
226
|
+
|
227
|
+
sock ||= HeadSock.start(uri)
|
228
|
+
sock.http_init(uri)
|
229
|
+
sock
|
230
|
+
rescue
|
231
|
+
# we'll return nil on any errors
|
232
|
+
end
|
233
|
+
|
234
|
+
# returns an idle socket to the pool
|
235
|
+
def sock_put(sock)
|
236
|
+
key = key_for(sock.uri)
|
237
|
+
sock.http_reusable? and @lock.synchronize { @pool[key] << sock }
|
238
|
+
rescue => err
|
239
|
+
error("HTTP reuse check failed: #{err.message} (#{err.class})")
|
240
|
+
end
|
241
|
+
end
|
data/omgf.gemspec
CHANGED
@@ -20,6 +20,8 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.rdoc_options = rdoc_options
|
21
21
|
s.test_files = Dir['test/test_*.rb']
|
22
22
|
s.add_dependency('rack', ['~> 1.3'])
|
23
|
+
s.add_dependency('kgio', ['~> 2.7'])
|
24
|
+
s.add_dependency('kcar', ['~> 0.3'])
|
23
25
|
s.add_dependency('mogilefs-client', ['~> 3.1'])
|
24
26
|
|
25
27
|
s.licenses = %w(AGPLv3+)
|
data/test/integration.rb
CHANGED
@@ -67,9 +67,9 @@ module TestMogileFSIntegration
|
|
67
67
|
[ status, out, err ]
|
68
68
|
end
|
69
69
|
|
70
|
-
def setup_mogilefs(plugins = nil)
|
70
|
+
def setup_mogilefs(plugins = nil, mogstored = "mogstored")
|
71
71
|
@test_host = "127.0.0.1"
|
72
|
-
setup_mogstored
|
72
|
+
setup_mogstored(mogstored)
|
73
73
|
@tracker = TCPServer.new(@test_host, 0)
|
74
74
|
@tracker_port = @tracker.addr[1]
|
75
75
|
|
@@ -173,7 +173,7 @@ EOF
|
|
173
173
|
raise "#{uri} failed to appear: #{res.inspect}"
|
174
174
|
end
|
175
175
|
|
176
|
-
def setup_mogstored
|
176
|
+
def setup_mogstored(mogstored = "mogstored")
|
177
177
|
@docroot = Dir.mktmpdir(["mogfresh", "docroot"])
|
178
178
|
Dir.mkdir("#@docroot/dev1")
|
179
179
|
Dir.mkdir("#@docroot/dev2")
|
@@ -194,7 +194,7 @@ EOF
|
|
194
194
|
@mogstored_mgmt.close
|
195
195
|
@mogstored_http.close
|
196
196
|
|
197
|
-
x!(
|
197
|
+
x!(mogstored, "--daemon", "--config=#{@mogstored_conf.path}")
|
198
198
|
wait_for_port @mogstored_mgmt_port
|
199
199
|
wait_for_port @mogstored_http_port
|
200
200
|
end
|
@@ -4,15 +4,17 @@ require './test/integration'
|
|
4
4
|
require 'rack/mock'
|
5
5
|
require 'open-uri'
|
6
6
|
require 'omgf/hysterical_raisins'
|
7
|
+
require 'digest/md5'
|
7
8
|
|
8
9
|
class TestHystericalRaisins < Test::Unit::TestCase
|
9
10
|
include TestMogileFSIntegration
|
10
11
|
def setup
|
11
12
|
setup_mogilefs
|
12
|
-
@app = OMGF::HystericalRaisins.new(:hosts => @hosts)
|
13
|
-
@req = Rack::MockRequest.new(@app)
|
14
13
|
@err = StringIO.new
|
15
|
-
|
14
|
+
logger = Logger.new(@err)
|
15
|
+
@opts = { "rack.logger" => logger }
|
16
|
+
@app = OMGF::HystericalRaisins.new(:hosts => @hosts, :logger => logger)
|
17
|
+
@req = Rack::MockRequest.new(@app)
|
16
18
|
@admin.create_domain("testdom")
|
17
19
|
end
|
18
20
|
|
@@ -186,13 +188,21 @@ class TestHystericalRaisins < Test::Unit::TestCase
|
|
186
188
|
|
187
189
|
# wait for replication
|
188
190
|
if ENV["EXPENSIVE"]
|
189
|
-
|
190
|
-
resp["X-URL-1"] and break
|
191
|
-
sleep 0.1
|
191
|
+
50.times do
|
192
192
|
resp = @req.head("/testdom/key", @opts)
|
193
|
+
assert_equal 200, resp.status, resp.inspect
|
194
|
+
resp["X-URL-1"] and break
|
195
|
+
sleep 0.5
|
193
196
|
end
|
194
|
-
assert_kind_of String, resp["X-URL-1"]
|
197
|
+
assert_kind_of String, resp["X-URL-1"], resp.inspect
|
195
198
|
assert_equal "BLAH", open(resp["X-URL-1"]).read
|
199
|
+
|
200
|
+
# Location should not be in X-Alt-Location, too
|
201
|
+
resp = @req.get("/testdom/key", @opts)
|
202
|
+
assert_equal 302, resp.status, resp.inspect
|
203
|
+
assert_kind_of URI, URI(resp["Location"])
|
204
|
+
assert_kind_of URI, URI(resp["X-Alt-Location-0"])
|
205
|
+
assert_nil resp["X-Alt-Location-1"]
|
196
206
|
end
|
197
207
|
|
198
208
|
reproxy_test
|
@@ -232,6 +242,24 @@ class TestHystericalRaisins < Test::Unit::TestCase
|
|
232
242
|
@app.instance_variable_set(:@reproxy_path, nil)
|
233
243
|
end
|
234
244
|
|
245
|
+
def test_checksums
|
246
|
+
mogadm!("class", "add", "testdom", "md5", "--hashtype=MD5")
|
247
|
+
100.times do
|
248
|
+
@admin.get_domains["testdom"]["md5"] and break
|
249
|
+
sleep 0.5
|
250
|
+
end
|
251
|
+
assert_equal "MD5", @admin.get_domains["testdom"]["md5"]["hashtype"]
|
252
|
+
sio = StringIO.new("HELLO")
|
253
|
+
md5 = [ Digest::MD5.digest(sio.string) ].pack('m0')
|
254
|
+
opts = @opts.merge(input: sio, method: "PUT", "HTTP_CONTENT_MD5" => md5)
|
255
|
+
resp = @req.put("/testdom/cc?class=md5", opts)
|
256
|
+
assert_equal 200, resp.status
|
257
|
+
@mg = MogileFS::MogileFS.new(hosts: @hosts, domain: "testdom")
|
258
|
+
info = @mg.file_info("cc")
|
259
|
+
assert_equal "md5", info["class"]
|
260
|
+
assert_equal "MD5:#{Digest::MD5.hexdigest(sio.string)}", info["checksum"]
|
261
|
+
end
|
262
|
+
|
235
263
|
def teardown
|
236
264
|
teardown_mogilefs
|
237
265
|
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
|
2
|
+
# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
|
3
|
+
require './test/integration'
|
4
|
+
require 'rack/mock'
|
5
|
+
require 'open-uri'
|
6
|
+
require 'omgf/hysterical_raisins'
|
7
|
+
require 'digest/md5'
|
8
|
+
|
9
|
+
class TestHystericalRaisinsCmogstored < Test::Unit::TestCase
|
10
|
+
include TestMogileFSIntegration
|
11
|
+
def setup
|
12
|
+
setup_mogilefs(nil, "cmogstored")
|
13
|
+
@err = StringIO.new
|
14
|
+
logger = Logger.new(@err)
|
15
|
+
@opts = { "rack.logger" => logger }
|
16
|
+
@app = OMGF::HystericalRaisins.new(:hosts => @hosts, :logger => logger)
|
17
|
+
@req = Rack::MockRequest.new(@app)
|
18
|
+
@admin.create_domain("testdom")
|
19
|
+
end
|
20
|
+
|
21
|
+
def test_checksums_trailer
|
22
|
+
mogadm!("class", "add", "testdom", "md5", "--hashtype=MD5")
|
23
|
+
100.times do
|
24
|
+
@admin.get_domains["testdom"]["md5"] and break
|
25
|
+
sleep 0.5
|
26
|
+
end
|
27
|
+
assert_equal "MD5", @admin.get_domains["testdom"]["md5"]["hashtype"]
|
28
|
+
|
29
|
+
sio = StringIO.new("HELLO")
|
30
|
+
def sio.read(*_)
|
31
|
+
rv = super
|
32
|
+
if rv == nil
|
33
|
+
md5 = [ Digest::MD5.digest(string) ].pack('m0')
|
34
|
+
@rack_env["HTTP_CONTENT_MD5"] = md5
|
35
|
+
end
|
36
|
+
rv
|
37
|
+
end
|
38
|
+
|
39
|
+
opts = @opts.merge(input: sio, method: "PUT")
|
40
|
+
opts["HTTP_TRAILER"] = "Content-MD5"
|
41
|
+
env = @req.class.env_for("/testdom/cc?class=md5", opts)
|
42
|
+
sio.instance_variable_set(:@rack_env, env)
|
43
|
+
status, _, _ = @app.call(env)
|
44
|
+
assert_equal 200, status, @err.string
|
45
|
+
@mg = MogileFS::MogileFS.new(hosts: @hosts, domain: "testdom")
|
46
|
+
info = @mg.file_info("cc")
|
47
|
+
assert_equal "md5", info["class"]
|
48
|
+
assert_equal "MD5:#{Digest::MD5.hexdigest(sio.string)}", info["checksum"]
|
49
|
+
end
|
50
|
+
|
51
|
+
def teardown
|
52
|
+
teardown_mogilefs
|
53
|
+
end
|
54
|
+
end if `which cmogstored 2>/dev/null` =~ /\bcmogstored\b/
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: !binary |-
|
3
3
|
b21nZg==
|
4
4
|
version: !ruby/object:Gem::Version
|
5
|
-
version: 0.0.
|
5
|
+
version: 0.0.1.GIT
|
6
6
|
prerelease: 6
|
7
7
|
platform: ruby
|
8
8
|
authors:
|
@@ -11,7 +11,7 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date: 2012-06-
|
14
|
+
date: 2012-06-23 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: !binary |-
|
@@ -34,6 +34,48 @@ dependencies:
|
|
34
34
|
- !ruby/object:Gem::Version
|
35
35
|
version: !binary |-
|
36
36
|
MS4z
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: !binary |-
|
39
|
+
a2dpbw==
|
40
|
+
requirement: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - !binary |-
|
44
|
+
fj4=
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: !binary |-
|
47
|
+
Mi43
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
none: false
|
52
|
+
requirements:
|
53
|
+
- - !binary |-
|
54
|
+
fj4=
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: !binary |-
|
57
|
+
Mi43
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: !binary |-
|
60
|
+
a2Nhcg==
|
61
|
+
requirement: !ruby/object:Gem::Requirement
|
62
|
+
none: false
|
63
|
+
requirements:
|
64
|
+
- - !binary |-
|
65
|
+
fj4=
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: !binary |-
|
68
|
+
MC4z
|
69
|
+
type: :runtime
|
70
|
+
prerelease: false
|
71
|
+
version_requirements: !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - !binary |-
|
75
|
+
fj4=
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: !binary |-
|
78
|
+
MC4z
|
37
79
|
- !ruby/object:Gem::Dependency
|
38
80
|
name: !binary |-
|
39
81
|
bW9naWxlZnMtY2xpZW50
|
@@ -71,6 +113,8 @@ extra_rdoc_files:
|
|
71
113
|
- lib/omgf.rb
|
72
114
|
- lib/omgf/hysterical_raisins.rb
|
73
115
|
- lib/omgf/pool.rb
|
116
|
+
- lib/omgf/verify_paths.rb
|
117
|
+
- lib/omgf/version.rb
|
74
118
|
- NEWS
|
75
119
|
- README
|
76
120
|
files:
|
@@ -88,14 +132,18 @@ files:
|
|
88
132
|
- README
|
89
133
|
- examples/hyst.README
|
90
134
|
- examples/hyst.bash
|
135
|
+
- examples/verify_paths_manual_test.rb
|
91
136
|
- lib/omgf.rb
|
92
137
|
- lib/omgf/hysterical_raisins.rb
|
93
138
|
- lib/omgf/pool.rb
|
139
|
+
- lib/omgf/verify_paths.rb
|
140
|
+
- lib/omgf/version.rb
|
94
141
|
- omgf.gemspec
|
95
142
|
- pkg.mk
|
96
143
|
- test/integration.rb
|
97
144
|
- test/test_hyst.rb
|
98
145
|
- test/test_hysterical_raisins.rb
|
146
|
+
- test/test_hysterical_raisins_cmogstored.rb
|
99
147
|
homepage: http://bogomips.org/omgf/
|
100
148
|
licenses:
|
101
149
|
- !binary |-
|
@@ -127,6 +175,8 @@ signing_key:
|
|
127
175
|
specification_version: 3
|
128
176
|
summary: hysterical REST API for MogileFS using Rack
|
129
177
|
test_files:
|
178
|
+
- !binary |-
|
179
|
+
dGVzdC90ZXN0X2h5c3RlcmljYWxfcmFpc2luc19jbW9nc3RvcmVkLnJi
|
130
180
|
- !binary |-
|
131
181
|
dGVzdC90ZXN0X2h5c3QucmI=
|
132
182
|
- !binary |-
|