omgf 0.0.0.GIT → 0.0.1.GIT

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore CHANGED
@@ -12,3 +12,4 @@ pkg/
12
12
  /man
13
13
  /tmp
14
14
  /LATEST
15
+ lib/omgf/version.rb
@@ -1,40 +1,30 @@
1
- #!/bin/sh
2
-
3
- GVF=GIT-VERSION-FILE
4
- DEF_VER=v0.0.0.GIT
5
-
6
- LF='
7
- '
1
+ #!/usr/bin/env ruby
2
+ CONSTANT = "OMGF::VERSION"
3
+ RVF = "lib/omgf/version.rb"
4
+ GVF = "GIT-VERSION-FILE"
5
+ DEF_VER = "v0.0.1.GIT"
6
+ vn = DEF_VER
8
7
 
9
8
  # First see if there is a version file (included in release tarballs),
10
9
  # then try git-describe, then default.
11
- if test -f version
12
- then
13
- VN=$(cat version) || VN="$DEF_VER"
14
- elif test -d .git -o -f .git &&
15
- VN=$(git describe --abbrev=4 HEAD 2>/dev/null) &&
16
- case "$VN" in
17
- *$LF*) (exit 1) ;;
18
- v[0-9]*)
19
- git update-index -q --refresh
20
- test -z "$(git diff-index --name-only HEAD --)" ||
21
- VN="$VN-dirty" ;;
22
- esac
23
- then
24
- VN=$(echo "$VN" | sed -e 's/-/./g');
25
- else
26
- VN="$DEF_VER"
27
- fi
28
-
29
- VN=$(expr "$VN" : v*'\(.*\)')
10
+ if File.exist?(".git")
11
+ describe = `git describe --abbrev=4 HEAD 2>/dev/null`.strip
12
+ case describe
13
+ when /\Av[0-9]*/
14
+ vn = describe
15
+ system(*%w(git update-index -q --refresh))
16
+ unless `git diff-index --name-only HEAD --`.chomp.empty?
17
+ vn << "-dirty"
18
+ end
19
+ vn.tr!('-', '.')
20
+ end
21
+ end
30
22
 
31
- if test -r $GVF
32
- then
33
- VC=$(sed -e 's/^GIT_VERSION = //' <$GVF)
34
- else
35
- VC=unset
36
- fi
37
- test "$VN" = "$VC" || {
38
- echo >&2 "GIT_VERSION = $VN"
39
- echo "GIT_VERSION = $VN" >$GVF
40
- }
23
+ vn = vn.sub!(/\Av/, "")
24
+ new_ruby_version = "#{CONSTANT} = '#{vn}'\n"
25
+ cur_ruby_version = File.read(RVF) rescue nil
26
+ if new_ruby_version != cur_ruby_version
27
+ File.open(GVF, "w") { |fp| fp.write("GIT_VERSION = #{vn}\n") }
28
+ File.open(RVF, "w") { |fp| fp.write(new_ruby_version) }
29
+ end
30
+ puts vn if $0 == __FILE__
@@ -2,4 +2,5 @@ all::
2
2
  RSYNC_DEST := bogomips.org:/srv/bogomips/omgf
3
3
  rfproject := rainbows
4
4
  rfpackage := omgf
5
+ pkg_extra += lib/omgf/version.rb
5
6
  include pkg.mk
@@ -0,0 +1,20 @@
1
+ # -*- encoding: binary -*-
2
+ require "omgf/verify_paths"
3
+ require "logger"
4
+ require "uri"
5
+ require "pp"
6
+ vp = OMGF::VerifyPaths.new(Logger.new($stderr))
7
+ urls = %w(
8
+ http://yhbt.net/
9
+ http://bogomips.org/omgf.git
10
+ http://bogomips.org/ozZZZZmgf.git
11
+ http://bogomips.org/ozZZZZ
12
+ http://bogomips.org/
13
+ http://127.0.0.1:666/
14
+ )
15
+ uris = urls.map { |uri| URI(uri) }
16
+
17
+ pp vp.verify(uris, 1, 0.1)
18
+ uris = urls.map { |uri| URI(uri) }
19
+ sleep 1
20
+ pp vp.verify(uris, 1, 0.1)
@@ -4,3 +4,4 @@
4
4
  # :startdoc:
5
5
  module OMGF
6
6
  end
7
+ require "omgf/version"
@@ -4,6 +4,7 @@
4
4
  # License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
5
5
  # :startdoc:
6
6
  require "omgf/pool"
7
+ require "omgf/verify_paths"
7
8
  require "time"
8
9
  require "json"
9
10
  require "mogilefs"
@@ -41,7 +42,15 @@ class OMGF::HystericalRaisins
41
42
  noverify: opts[:noverify],
42
43
  pathcount: opts[:pathcount] || 0x7fffffff,
43
44
  }
45
+ @new_file_opts = {
46
+ content_md5: opts[:content_md5], # :trailer is acceptable here
47
+
48
+ # largefile: auto-selects based on env["CONTENT_LENGTH"]
49
+ largefile: opts[:largefile] || :stream,
50
+ }
44
51
  @put_overwrite_header = opts[:put_overwrite_header] || "HTTP_X_OMGF_FORCE"
52
+ @vp = OMGF::VerifyPaths.new(opts[:logger])
53
+ @verify_timeout = opts[:verify_timeout] || 0.5
45
54
  pool_init(mg_opts)
46
55
  end
47
56
 
@@ -98,13 +107,16 @@ class OMGF::HystericalRaisins
98
107
 
99
108
  # HEAD /$DOMAIN/$KEY
100
109
  def stat_key(env, domain, key)
101
- size, paths = mg_size_and_paths(domain, key, @get_paths_opts)
102
- return r(503, "") unless paths && location = paths[0]
110
+ size, uris = mg_size_and_uris(domain, key, @get_paths_opts)
111
+ uris = @vp.verify(uris, 1, @verify_timeout).flatten!
112
+
113
+ return r(503, "") unless uris && uris[0]
114
+
103
115
  h = { "Content-Length" => size.to_s }
104
116
  fn = filename(h, query(env)) || key
105
117
  h["Content-Type"] = key_mime_type(fn)
106
- unless reproxy?(env, key, h, location)
107
- paths.each_with_index { |path,i| h["X-URL-#{i}"] = path }
118
+ unless reproxy?(env, key, h, uris[0].to_s)
119
+ uris.each_with_index { |uri,i| h["X-URL-#{i}"] = uri.to_s }
108
120
  end
109
121
  [ 200, h, [] ]
110
122
  end
@@ -112,6 +124,7 @@ class OMGF::HystericalRaisins
112
124
  # GET /$DOMAIN/$KEY
113
125
  def redirect_key(env, domain, key)
114
126
  uris = mg_get_uris(domain, key, @get_paths_opts)
127
+ uris = @vp.verify(uris, 1, @verify_timeout).flatten!
115
128
 
116
129
  return r(503, "") unless uris && dest = uris.shift
117
130
 
@@ -123,7 +136,6 @@ class OMGF::HystericalRaisins
123
136
  }
124
137
 
125
138
  unless reproxy?(env, key, h, location)
126
- uris -= [ dest ]
127
139
  uris.each_with_index { |uri,i| h["X-Alt-Location-#{i}"] = uri.to_s }
128
140
  end
129
141
  [ 302, h, [] ]
@@ -252,16 +264,39 @@ class OMGF::HystericalRaisins
252
264
  return r(406, "key `#{key}' is not URI-friendly") if bad_key?(key)
253
265
  return r(406, "key is too long") if key.size > 128
254
266
 
267
+ clen = env["CONTENT_LENGTH"]
268
+
255
269
  # this was written before MogileFS supported empty files,
256
270
  # but empty files waste DB space so we don't support them
257
271
  # Not bothering with Transfer-Encoding: chunked, though...
258
- return r(403, "empty files forbidden") if env["CONTENT_LENGTH"] == "0"
272
+ return r(403, "empty files forbidden") if "0" == clen
259
273
 
260
274
  params = query(env)
261
275
  input = env["rack.input"]
262
276
  paths = nil
263
277
  retried = false
264
278
 
279
+ # prepare options for create_open/create_close:
280
+ new_file_opts = @new_file_opts.dup
281
+
282
+ # the original deployment of this created a class for every
283
+ # domain with the same class having the same name as the domain
284
+ new_file_opts[:class] = params['class'] || @default_class_cb[domain]
285
+
286
+ # try to give a Content-Length to the tracker
287
+ clen and new_file_opts[:content_length] = clen.to_i
288
+
289
+ if /\bContent-MD5\b/i =~ env["HTTP_TRAILER"]
290
+ # if the client will give the Content-MD5 as the trailer,
291
+ # we must lazily populate it since we're not guaranteed to
292
+ # have the trailer, yet (rack.input is lazily read on unicorn)
293
+ new_file_opts[:content_md5] = lambda { env["HTTP_CONTENT_MD5"] }
294
+ elsif cmd5 = env["HTTP_CONTENT_MD5"]
295
+ # maybe the client gave the Content-MD5 in the header
296
+ new_file_opts[:content_md5] = cmd5
297
+ end
298
+
299
+ buf = ""
265
300
  begin
266
301
  pool_use(domain) do |mg|
267
302
  begin
@@ -281,14 +316,17 @@ class OMGF::HystericalRaisins
281
316
  # good, not clobbering anything
282
317
  end
283
318
 
284
- # the original deployment of this created a class for every
285
- # domain with the same class having the same name as the domain
286
- klass = params['class'] || @default_class_cb[domain] || "default"
287
- mg.store_file(key, klass, input)
319
+ # finally, upload the file
320
+ mg.new_file(key, new_file_opts) do |io|
321
+ while input.read(16384, buf)
322
+ io.write(buf)
323
+ end
324
+ end
288
325
  end # pool_use
289
326
 
290
327
  # should always return 201 if ! found, but we keep 200 for legacy
291
328
  # compat if they're not logged in (via REMOTE_USER)
329
+ buf.clear
292
330
  status = paths ? 204 : (env["REMOTE_USER"] ? 201 : 200)
293
331
  r(status, "")
294
332
  rescue MogileFS::Backend::UnregDomainError,
@@ -40,9 +40,9 @@ module OMGF::Pool
40
40
  pool_use(domain) { |mg| mg.get_uris(key, opts) }
41
41
  end
42
42
 
43
- def mg_size_and_paths(domain, key, opts) # :nodoc:
43
+ def mg_size_and_uris(domain, key, opts) # :nodoc:
44
44
  pool_use(domain) do |mg|
45
- [ mg.size(key), mg.get_paths(key, opts) ]
45
+ [ mg.size(key), mg.get_uris(key, opts) ]
46
46
  end
47
47
  end
48
48
  end
@@ -0,0 +1,241 @@
1
+ # -*- encoding: binary -*-
2
+ # :stopdoc:
3
+ # Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
4
+ # License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
5
+ # :startdoc:
6
+ require "thread"
7
+ require "omgf"
8
+ require "kcar"
9
+ require "kgio"
10
+
11
+ # This module makes HEAD requests with reusable HTTP connections to
12
+ # verify paths. This is faster than having the mogilefsd tracker
13
+ # verifying paths, and the client could have broken routing to some
14
+ # the storage nodes the mogilefsd tracker/monitor can see.
15
+ class OMGF::VerifyPaths
16
+
17
+ # private class
18
+ class HeadSock < Kgio::Socket # :nodoc:
19
+ VERSION = '1.0.0'
20
+
21
+ attr_reader :uri
22
+ attr_writer :retry_ok
23
+
24
+ def self.start(uri)
25
+ super(Socket.pack_sockaddr_in(uri.port, uri.host))
26
+ end
27
+
28
+ # returns true if the HTTP connection is reusable
29
+ def http_reusable?
30
+ rv = @kcar.keepalive?
31
+ @kcar.reset
32
+ @headers.clear
33
+ @buf.clear
34
+ @uri = nil
35
+ rv ? setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 1) : close
36
+ rv
37
+ end
38
+
39
+ def http_init(uri, retry_ok = true)
40
+ @uri = uri
41
+ unless defined?(@kcar)
42
+ @kcar = Kcar::Parser.new
43
+ @headers = {}
44
+ @buf = ""
45
+ end
46
+ @retry_ok = retry_ok
47
+
48
+ # prepare the HTTP request
49
+ @req = "HEAD #{@uri.request_uri} HTTP/1.1\r\n" \
50
+ "User-Agent: #{self.class}/#{VERSION}\r\n" \
51
+ "Host: #{@uri.host}:#{@uri.port}\r\n" \
52
+ "\r\n"
53
+ end
54
+
55
+ # returns an array result if successful
56
+ # returns :wait_readable or :wait_writable if incomplete
57
+ # returns a subclass of Exception on errors
58
+ # returns nil on premature EOF (twice)
59
+ def poll_iter(pollset)
60
+ case rv = kgio_trywrite(@req)
61
+ when nil, # done writing, start reading
62
+ String # partial write
63
+ @req = rv # continue looping
64
+ when Symbol
65
+ return pollset[self] = rv # busy
66
+ end while @req
67
+
68
+ case rv = kgio_tryread(666)
69
+ when String
70
+ if ary = @kcar.headers(@headers, @buf << rv)
71
+ pollset.delete(self)
72
+ return ary # success
73
+ end
74
+ # continue looping if incomplete
75
+ when Symbol
76
+ return pollset[self] = rv # busy
77
+ when nil # EOF (premature)
78
+ return maybe_retry(nil, pollset)
79
+ end while true
80
+ rescue => err
81
+ maybe_retry(err, pollset)
82
+ end
83
+
84
+ # returns the err object if we've already retried, nil otherwise
85
+ def maybe_retry(err, pollset)
86
+ pollset.delete(self)
87
+ return err unless @retry_ok
88
+
89
+ # always start a fresh connection on socket errors
90
+ sock = self.class.start(@uri)
91
+ sock.http_init(@uri, false)
92
+ pollset[sock] = :wait_writable
93
+ end
94
+ end
95
+
96
+ def initialize(logger)
97
+ @pool = Hash.new { |hash,host_port| hash[host_port] = [] }
98
+ @logger = logger
99
+ @finishq = Queue.new
100
+ @finisher = nil
101
+ @lock = Mutex.new
102
+ @pid = $$
103
+ end
104
+
105
+ def error(msg)
106
+ @logger.error(msg) if @logger
107
+ end
108
+
109
+ def iter_check(ok, sock, pollset)
110
+ rv = sock.poll_iter(pollset)
111
+ case rv
112
+ when Symbol # in progress
113
+ when Array
114
+ code = rv[0].to_i
115
+ if 200 == code
116
+ ok << sock.uri
117
+ elsif code >= 100 && code <= 999
118
+ error("HEAD #{sock.uri} returned HTTP code: #{code}")
119
+ else
120
+ error("HEAD #{sock.uri} returned #{rv.inspect} (kcar bug?)")
121
+ end
122
+ sock_put(sock)
123
+ when nil # premature EOF
124
+ error("HEAD #{sock.uri} hit socket EOF")
125
+ else
126
+ # exception or some other error return value...
127
+ if rv.respond_to?(:message)
128
+ error("HEAD #{sock.uri} error: #{rv.message} (#{rv.class})")
129
+ else
130
+ error("HEAD #{sock.uri} error (#{rv.class}): #{rv.inspect}")
131
+ end
132
+ end
133
+ rv
134
+ end
135
+
136
+ # this runs in a background thread to cleanup all the requests
137
+ # that didn't finish quickly enough
138
+ def finisher(timeout = 10000)
139
+ begin
140
+ pollset = @finishq.pop # park here when idle
141
+
142
+ while ready = Kgio.poll(pollset.dup, timeout)
143
+ ready.each_key do |sock|
144
+ sock.retry_ok = false
145
+ # try to return good sockets back to the pool
146
+ iter_check([], sock, pollset)
147
+ end
148
+
149
+ # try to stuff the pollset as much as possible for further looping
150
+ while more = (@finishq.pop(true) rescue nil)
151
+ pollset.merge!(more)
152
+ end
153
+ end
154
+
155
+ # connections timed out, kill them
156
+ pollset.each_key { |sock| sock.close }
157
+ rescue => err
158
+ error("#{err.message} (#{err.class})")
159
+ end while true
160
+ end
161
+
162
+ # reorders URIs based on response time
163
+ # This is the main method of this class
164
+ def verify(uris, count, timeout)
165
+ tout = (timeout * 1000).to_i
166
+ pollset = {}
167
+ ok = []
168
+
169
+ uris.each do |uri|
170
+ sock = sock_get(uri) and iter_check(ok, sock, pollset)
171
+ end
172
+
173
+ while ok.size < count && tout > 0 && ! pollset.empty?
174
+ t0 = Time.now
175
+ ready = Kgio.poll(pollset.dup, tout) or break
176
+ tout -= ((Time.now - t0) * 1000).to_i
177
+
178
+ ready.each_key do |sock|
179
+ iter_check(ok, sock, pollset)
180
+ end
181
+ end
182
+
183
+ finish(pollset) unless pollset.empty?
184
+ [ok, uris - ok] # good URLs first
185
+ end
186
+
187
+ # recover any unfinished URLs in pollset asynchronously in the background
188
+ def finish(pollset) # :nodoc:
189
+ @finishq.push(pollset)
190
+ @lock.synchronize do
191
+ unless @finisher && @finisher.alive?
192
+ @finisher = Thread.new { finisher }
193
+ end
194
+ end
195
+ end
196
+
197
+ # returns a string key for the connection pool
198
+ def key_for(uri)
199
+ "#{uri.host}:#{uri.port}"
200
+ end
201
+
202
+ # initializes a cached connection for +uri+ or creates a new one
203
+ def sock_get(uri)
204
+ key = key_for(uri)
205
+
206
+ # detect forks and prevent sharing of connected sockets across processes
207
+ @lock.synchronize do
208
+ if @pid != $$
209
+ @pid = $$
210
+ @pool.clear
211
+ end
212
+ end
213
+
214
+ while sock = @lock.synchronize { @pool[key].pop }
215
+ begin
216
+ # check if sock is still alive and idle
217
+ # :wait_readable is good here
218
+ break if sock.kgio_tryread(1) == :wait_readable
219
+ rescue
220
+ # ignore socket errors, we'll just give them a new socket
221
+ # socket should've been idle, but it was not (or EOFed on us)
222
+ # give them a new one
223
+ end
224
+ sock.close
225
+ end
226
+
227
+ sock ||= HeadSock.start(uri)
228
+ sock.http_init(uri)
229
+ sock
230
+ rescue
231
+ # we'll return nil on any errors
232
+ end
233
+
234
+ # returns an idle socket to the pool
235
+ def sock_put(sock)
236
+ key = key_for(sock.uri)
237
+ sock.http_reusable? and @lock.synchronize { @pool[key] << sock }
238
+ rescue => err
239
+ error("HTTP reuse check failed: #{err.message} (#{err.class})")
240
+ end
241
+ end
@@ -20,6 +20,8 @@ Gem::Specification.new do |s|
20
20
  s.rdoc_options = rdoc_options
21
21
  s.test_files = Dir['test/test_*.rb']
22
22
  s.add_dependency('rack', ['~> 1.3'])
23
+ s.add_dependency('kgio', ['~> 2.7'])
24
+ s.add_dependency('kcar', ['~> 0.3'])
23
25
  s.add_dependency('mogilefs-client', ['~> 3.1'])
24
26
 
25
27
  s.licenses = %w(AGPLv3+)
@@ -67,9 +67,9 @@ module TestMogileFSIntegration
67
67
  [ status, out, err ]
68
68
  end
69
69
 
70
- def setup_mogilefs(plugins = nil)
70
+ def setup_mogilefs(plugins = nil, mogstored = "mogstored")
71
71
  @test_host = "127.0.0.1"
72
- setup_mogstored
72
+ setup_mogstored(mogstored)
73
73
  @tracker = TCPServer.new(@test_host, 0)
74
74
  @tracker_port = @tracker.addr[1]
75
75
 
@@ -173,7 +173,7 @@ EOF
173
173
  raise "#{uri} failed to appear: #{res.inspect}"
174
174
  end
175
175
 
176
- def setup_mogstored
176
+ def setup_mogstored(mogstored = "mogstored")
177
177
  @docroot = Dir.mktmpdir(["mogfresh", "docroot"])
178
178
  Dir.mkdir("#@docroot/dev1")
179
179
  Dir.mkdir("#@docroot/dev2")
@@ -194,7 +194,7 @@ EOF
194
194
  @mogstored_mgmt.close
195
195
  @mogstored_http.close
196
196
 
197
- x!("mogstored", "--daemon", "--config=#{@mogstored_conf.path}")
197
+ x!(mogstored, "--daemon", "--config=#{@mogstored_conf.path}")
198
198
  wait_for_port @mogstored_mgmt_port
199
199
  wait_for_port @mogstored_http_port
200
200
  end
@@ -4,15 +4,17 @@ require './test/integration'
4
4
  require 'rack/mock'
5
5
  require 'open-uri'
6
6
  require 'omgf/hysterical_raisins'
7
+ require 'digest/md5'
7
8
 
8
9
  class TestHystericalRaisins < Test::Unit::TestCase
9
10
  include TestMogileFSIntegration
10
11
  def setup
11
12
  setup_mogilefs
12
- @app = OMGF::HystericalRaisins.new(:hosts => @hosts)
13
- @req = Rack::MockRequest.new(@app)
14
13
  @err = StringIO.new
15
- @opts = { "rack.logger" => Logger.new(@err) }
14
+ logger = Logger.new(@err)
15
+ @opts = { "rack.logger" => logger }
16
+ @app = OMGF::HystericalRaisins.new(:hosts => @hosts, :logger => logger)
17
+ @req = Rack::MockRequest.new(@app)
16
18
  @admin.create_domain("testdom")
17
19
  end
18
20
 
@@ -186,13 +188,21 @@ class TestHystericalRaisins < Test::Unit::TestCase
186
188
 
187
189
  # wait for replication
188
190
  if ENV["EXPENSIVE"]
189
- 200.times do
190
- resp["X-URL-1"] and break
191
- sleep 0.1
191
+ 50.times do
192
192
  resp = @req.head("/testdom/key", @opts)
193
+ assert_equal 200, resp.status, resp.inspect
194
+ resp["X-URL-1"] and break
195
+ sleep 0.5
193
196
  end
194
- assert_kind_of String, resp["X-URL-1"]
197
+ assert_kind_of String, resp["X-URL-1"], resp.inspect
195
198
  assert_equal "BLAH", open(resp["X-URL-1"]).read
199
+
200
+ # Location should not be in X-Alt-Location, too
201
+ resp = @req.get("/testdom/key", @opts)
202
+ assert_equal 302, resp.status, resp.inspect
203
+ assert_kind_of URI, URI(resp["Location"])
204
+ assert_kind_of URI, URI(resp["X-Alt-Location-0"])
205
+ assert_nil resp["X-Alt-Location-1"]
196
206
  end
197
207
 
198
208
  reproxy_test
@@ -232,6 +242,24 @@ class TestHystericalRaisins < Test::Unit::TestCase
232
242
  @app.instance_variable_set(:@reproxy_path, nil)
233
243
  end
234
244
 
245
+ def test_checksums
246
+ mogadm!("class", "add", "testdom", "md5", "--hashtype=MD5")
247
+ 100.times do
248
+ @admin.get_domains["testdom"]["md5"] and break
249
+ sleep 0.5
250
+ end
251
+ assert_equal "MD5", @admin.get_domains["testdom"]["md5"]["hashtype"]
252
+ sio = StringIO.new("HELLO")
253
+ md5 = [ Digest::MD5.digest(sio.string) ].pack('m0')
254
+ opts = @opts.merge(input: sio, method: "PUT", "HTTP_CONTENT_MD5" => md5)
255
+ resp = @req.put("/testdom/cc?class=md5", opts)
256
+ assert_equal 200, resp.status
257
+ @mg = MogileFS::MogileFS.new(hosts: @hosts, domain: "testdom")
258
+ info = @mg.file_info("cc")
259
+ assert_equal "md5", info["class"]
260
+ assert_equal "MD5:#{Digest::MD5.hexdigest(sio.string)}", info["checksum"]
261
+ end
262
+
235
263
  def teardown
236
264
  teardown_mogilefs
237
265
  end
@@ -0,0 +1,54 @@
1
+ # Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
2
+ # License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
3
+ require './test/integration'
4
+ require 'rack/mock'
5
+ require 'open-uri'
6
+ require 'omgf/hysterical_raisins'
7
+ require 'digest/md5'
8
+
9
+ class TestHystericalRaisinsCmogstored < Test::Unit::TestCase
10
+ include TestMogileFSIntegration
11
+ def setup
12
+ setup_mogilefs(nil, "cmogstored")
13
+ @err = StringIO.new
14
+ logger = Logger.new(@err)
15
+ @opts = { "rack.logger" => logger }
16
+ @app = OMGF::HystericalRaisins.new(:hosts => @hosts, :logger => logger)
17
+ @req = Rack::MockRequest.new(@app)
18
+ @admin.create_domain("testdom")
19
+ end
20
+
21
+ def test_checksums_trailer
22
+ mogadm!("class", "add", "testdom", "md5", "--hashtype=MD5")
23
+ 100.times do
24
+ @admin.get_domains["testdom"]["md5"] and break
25
+ sleep 0.5
26
+ end
27
+ assert_equal "MD5", @admin.get_domains["testdom"]["md5"]["hashtype"]
28
+
29
+ sio = StringIO.new("HELLO")
30
+ def sio.read(*_)
31
+ rv = super
32
+ if rv == nil
33
+ md5 = [ Digest::MD5.digest(string) ].pack('m0')
34
+ @rack_env["HTTP_CONTENT_MD5"] = md5
35
+ end
36
+ rv
37
+ end
38
+
39
+ opts = @opts.merge(input: sio, method: "PUT")
40
+ opts["HTTP_TRAILER"] = "Content-MD5"
41
+ env = @req.class.env_for("/testdom/cc?class=md5", opts)
42
+ sio.instance_variable_set(:@rack_env, env)
43
+ status, _, _ = @app.call(env)
44
+ assert_equal 200, status, @err.string
45
+ @mg = MogileFS::MogileFS.new(hosts: @hosts, domain: "testdom")
46
+ info = @mg.file_info("cc")
47
+ assert_equal "md5", info["class"]
48
+ assert_equal "MD5:#{Digest::MD5.hexdigest(sio.string)}", info["checksum"]
49
+ end
50
+
51
+ def teardown
52
+ teardown_mogilefs
53
+ end
54
+ end if `which cmogstored 2>/dev/null` =~ /\bcmogstored\b/
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: !binary |-
3
3
  b21nZg==
4
4
  version: !ruby/object:Gem::Version
5
- version: 0.0.0.GIT
5
+ version: 0.0.1.GIT
6
6
  prerelease: 6
7
7
  platform: ruby
8
8
  authors:
@@ -11,7 +11,7 @@ authors:
11
11
  autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2012-06-20 00:00:00.000000000 Z
14
+ date: 2012-06-23 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: !binary |-
@@ -34,6 +34,48 @@ dependencies:
34
34
  - !ruby/object:Gem::Version
35
35
  version: !binary |-
36
36
  MS4z
37
+ - !ruby/object:Gem::Dependency
38
+ name: !binary |-
39
+ a2dpbw==
40
+ requirement: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - !binary |-
44
+ fj4=
45
+ - !ruby/object:Gem::Version
46
+ version: !binary |-
47
+ Mi43
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ none: false
52
+ requirements:
53
+ - - !binary |-
54
+ fj4=
55
+ - !ruby/object:Gem::Version
56
+ version: !binary |-
57
+ Mi43
58
+ - !ruby/object:Gem::Dependency
59
+ name: !binary |-
60
+ a2Nhcg==
61
+ requirement: !ruby/object:Gem::Requirement
62
+ none: false
63
+ requirements:
64
+ - - !binary |-
65
+ fj4=
66
+ - !ruby/object:Gem::Version
67
+ version: !binary |-
68
+ MC4z
69
+ type: :runtime
70
+ prerelease: false
71
+ version_requirements: !ruby/object:Gem::Requirement
72
+ none: false
73
+ requirements:
74
+ - - !binary |-
75
+ fj4=
76
+ - !ruby/object:Gem::Version
77
+ version: !binary |-
78
+ MC4z
37
79
  - !ruby/object:Gem::Dependency
38
80
  name: !binary |-
39
81
  bW9naWxlZnMtY2xpZW50
@@ -71,6 +113,8 @@ extra_rdoc_files:
71
113
  - lib/omgf.rb
72
114
  - lib/omgf/hysterical_raisins.rb
73
115
  - lib/omgf/pool.rb
116
+ - lib/omgf/verify_paths.rb
117
+ - lib/omgf/version.rb
74
118
  - NEWS
75
119
  - README
76
120
  files:
@@ -88,14 +132,18 @@ files:
88
132
  - README
89
133
  - examples/hyst.README
90
134
  - examples/hyst.bash
135
+ - examples/verify_paths_manual_test.rb
91
136
  - lib/omgf.rb
92
137
  - lib/omgf/hysterical_raisins.rb
93
138
  - lib/omgf/pool.rb
139
+ - lib/omgf/verify_paths.rb
140
+ - lib/omgf/version.rb
94
141
  - omgf.gemspec
95
142
  - pkg.mk
96
143
  - test/integration.rb
97
144
  - test/test_hyst.rb
98
145
  - test/test_hysterical_raisins.rb
146
+ - test/test_hysterical_raisins_cmogstored.rb
99
147
  homepage: http://bogomips.org/omgf/
100
148
  licenses:
101
149
  - !binary |-
@@ -127,6 +175,8 @@ signing_key:
127
175
  specification_version: 3
128
176
  summary: hysterical REST API for MogileFS using Rack
129
177
  test_files:
178
+ - !binary |-
179
+ dGVzdC90ZXN0X2h5c3RlcmljYWxfcmFpc2luc19jbW9nc3RvcmVkLnJi
130
180
  - !binary |-
131
181
  dGVzdC90ZXN0X2h5c3QucmI=
132
182
  - !binary |-