omgf 0.0.0.GIT → 0.0.1.GIT

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore CHANGED
@@ -12,3 +12,4 @@ pkg/
12
12
  /man
13
13
  /tmp
14
14
  /LATEST
15
+ lib/omgf/version.rb
@@ -1,40 +1,30 @@
1
- #!/bin/sh
2
-
3
- GVF=GIT-VERSION-FILE
4
- DEF_VER=v0.0.0.GIT
5
-
6
- LF='
7
- '
1
+ #!/usr/bin/env ruby
2
+ CONSTANT = "OMGF::VERSION"
3
+ RVF = "lib/omgf/version.rb"
4
+ GVF = "GIT-VERSION-FILE"
5
+ DEF_VER = "v0.0.1.GIT"
6
+ vn = DEF_VER
8
7
 
9
8
  # First see if there is a version file (included in release tarballs),
10
9
  # then try git-describe, then default.
11
- if test -f version
12
- then
13
- VN=$(cat version) || VN="$DEF_VER"
14
- elif test -d .git -o -f .git &&
15
- VN=$(git describe --abbrev=4 HEAD 2>/dev/null) &&
16
- case "$VN" in
17
- *$LF*) (exit 1) ;;
18
- v[0-9]*)
19
- git update-index -q --refresh
20
- test -z "$(git diff-index --name-only HEAD --)" ||
21
- VN="$VN-dirty" ;;
22
- esac
23
- then
24
- VN=$(echo "$VN" | sed -e 's/-/./g');
25
- else
26
- VN="$DEF_VER"
27
- fi
28
-
29
- VN=$(expr "$VN" : v*'\(.*\)')
10
+ if File.exist?(".git")
11
+ describe = `git describe --abbrev=4 HEAD 2>/dev/null`.strip
12
+ case describe
13
+ when /\Av[0-9]*/
14
+ vn = describe
15
+ system(*%w(git update-index -q --refresh))
16
+ unless `git diff-index --name-only HEAD --`.chomp.empty?
17
+ vn << "-dirty"
18
+ end
19
+ vn.tr!('-', '.')
20
+ end
21
+ end
30
22
 
31
- if test -r $GVF
32
- then
33
- VC=$(sed -e 's/^GIT_VERSION = //' <$GVF)
34
- else
35
- VC=unset
36
- fi
37
- test "$VN" = "$VC" || {
38
- echo >&2 "GIT_VERSION = $VN"
39
- echo "GIT_VERSION = $VN" >$GVF
40
- }
23
+ vn = vn.sub!(/\Av/, "")
24
+ new_ruby_version = "#{CONSTANT} = '#{vn}'\n"
25
+ cur_ruby_version = File.read(RVF) rescue nil
26
+ if new_ruby_version != cur_ruby_version
27
+ File.open(GVF, "w") { |fp| fp.write("GIT_VERSION = #{vn}\n") }
28
+ File.open(RVF, "w") { |fp| fp.write(new_ruby_version) }
29
+ end
30
+ puts vn if $0 == __FILE__
@@ -2,4 +2,5 @@ all::
2
2
  RSYNC_DEST := bogomips.org:/srv/bogomips/omgf
3
3
  rfproject := rainbows
4
4
  rfpackage := omgf
5
+ pkg_extra += lib/omgf/version.rb
5
6
  include pkg.mk
@@ -0,0 +1,20 @@
1
+ # -*- encoding: binary -*-
2
+ require "omgf/verify_paths"
3
+ require "logger"
4
+ require "uri"
5
+ require "pp"
6
+ vp = OMGF::VerifyPaths.new(Logger.new($stderr))
7
+ urls = %w(
8
+ http://yhbt.net/
9
+ http://bogomips.org/omgf.git
10
+ http://bogomips.org/ozZZZZmgf.git
11
+ http://bogomips.org/ozZZZZ
12
+ http://bogomips.org/
13
+ http://127.0.0.1:666/
14
+ )
15
+ uris = urls.map { |uri| URI(uri) }
16
+
17
+ pp vp.verify(uris, 1, 0.1)
18
+ uris = urls.map { |uri| URI(uri) }
19
+ sleep 1
20
+ pp vp.verify(uris, 1, 0.1)
@@ -4,3 +4,4 @@
4
4
  # :startdoc:
5
5
  module OMGF
6
6
  end
7
+ require "omgf/version"
@@ -4,6 +4,7 @@
4
4
  # License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
5
5
  # :startdoc:
6
6
  require "omgf/pool"
7
+ require "omgf/verify_paths"
7
8
  require "time"
8
9
  require "json"
9
10
  require "mogilefs"
@@ -41,7 +42,15 @@ class OMGF::HystericalRaisins
41
42
  noverify: opts[:noverify],
42
43
  pathcount: opts[:pathcount] || 0x7fffffff,
43
44
  }
45
+ @new_file_opts = {
46
+ content_md5: opts[:content_md5], # :trailer is acceptable here
47
+
48
+ # largefile: auto-selects based on env["CONTENT_LENGTH"]
49
+ largefile: opts[:largefile] || :stream,
50
+ }
44
51
  @put_overwrite_header = opts[:put_overwrite_header] || "HTTP_X_OMGF_FORCE"
52
+ @vp = OMGF::VerifyPaths.new(opts[:logger])
53
+ @verify_timeout = opts[:verify_timeout] || 0.5
45
54
  pool_init(mg_opts)
46
55
  end
47
56
 
@@ -98,13 +107,16 @@ class OMGF::HystericalRaisins
98
107
 
99
108
  # HEAD /$DOMAIN/$KEY
100
109
  def stat_key(env, domain, key)
101
- size, paths = mg_size_and_paths(domain, key, @get_paths_opts)
102
- return r(503, "") unless paths && location = paths[0]
110
+ size, uris = mg_size_and_uris(domain, key, @get_paths_opts)
111
+ uris = @vp.verify(uris, 1, @verify_timeout).flatten!
112
+
113
+ return r(503, "") unless uris && uris[0]
114
+
103
115
  h = { "Content-Length" => size.to_s }
104
116
  fn = filename(h, query(env)) || key
105
117
  h["Content-Type"] = key_mime_type(fn)
106
- unless reproxy?(env, key, h, location)
107
- paths.each_with_index { |path,i| h["X-URL-#{i}"] = path }
118
+ unless reproxy?(env, key, h, uris[0].to_s)
119
+ uris.each_with_index { |uri,i| h["X-URL-#{i}"] = uri.to_s }
108
120
  end
109
121
  [ 200, h, [] ]
110
122
  end
@@ -112,6 +124,7 @@ class OMGF::HystericalRaisins
112
124
  # GET /$DOMAIN/$KEY
113
125
  def redirect_key(env, domain, key)
114
126
  uris = mg_get_uris(domain, key, @get_paths_opts)
127
+ uris = @vp.verify(uris, 1, @verify_timeout).flatten!
115
128
 
116
129
  return r(503, "") unless uris && dest = uris.shift
117
130
 
@@ -123,7 +136,6 @@ class OMGF::HystericalRaisins
123
136
  }
124
137
 
125
138
  unless reproxy?(env, key, h, location)
126
- uris -= [ dest ]
127
139
  uris.each_with_index { |uri,i| h["X-Alt-Location-#{i}"] = uri.to_s }
128
140
  end
129
141
  [ 302, h, [] ]
@@ -252,16 +264,39 @@ class OMGF::HystericalRaisins
252
264
  return r(406, "key `#{key}' is not URI-friendly") if bad_key?(key)
253
265
  return r(406, "key is too long") if key.size > 128
254
266
 
267
+ clen = env["CONTENT_LENGTH"]
268
+
255
269
  # this was written before MogileFS supported empty files,
256
270
  # but empty files waste DB space so we don't support them
257
271
  # Not bothering with Transfer-Encoding: chunked, though...
258
- return r(403, "empty files forbidden") if env["CONTENT_LENGTH"] == "0"
272
+ return r(403, "empty files forbidden") if "0" == clen
259
273
 
260
274
  params = query(env)
261
275
  input = env["rack.input"]
262
276
  paths = nil
263
277
  retried = false
264
278
 
279
+ # prepare options for create_open/create_close:
280
+ new_file_opts = @new_file_opts.dup
281
+
282
+ # the original deployment of this created a class for every
283
+ # domain with the same class having the same name as the domain
284
+ new_file_opts[:class] = params['class'] || @default_class_cb[domain]
285
+
286
+ # try to give a Content-Length to the tracker
287
+ clen and new_file_opts[:content_length] = clen.to_i
288
+
289
+ if /\bContent-MD5\b/i =~ env["HTTP_TRAILER"]
290
+ # if the client will give the Content-MD5 as the trailer,
291
+ # we must lazily populate it since we're not guaranteed to
292
+ # have the trailer, yet (rack.input is lazily read on unicorn)
293
+ new_file_opts[:content_md5] = lambda { env["HTTP_CONTENT_MD5"] }
294
+ elsif cmd5 = env["HTTP_CONTENT_MD5"]
295
+ # maybe the client gave the Content-MD5 in the header
296
+ new_file_opts[:content_md5] = cmd5
297
+ end
298
+
299
+ buf = ""
265
300
  begin
266
301
  pool_use(domain) do |mg|
267
302
  begin
@@ -281,14 +316,17 @@ class OMGF::HystericalRaisins
281
316
  # good, not clobbering anything
282
317
  end
283
318
 
284
- # the original deployment of this created a class for every
285
- # domain with the same class having the same name as the domain
286
- klass = params['class'] || @default_class_cb[domain] || "default"
287
- mg.store_file(key, klass, input)
319
+ # finally, upload the file
320
+ mg.new_file(key, new_file_opts) do |io|
321
+ while input.read(16384, buf)
322
+ io.write(buf)
323
+ end
324
+ end
288
325
  end # pool_use
289
326
 
290
327
  # should always return 201 if ! found, but we keep 200 for legacy
291
328
  # compat if they're not logged in (via REMOTE_USER)
329
+ buf.clear
292
330
  status = paths ? 204 : (env["REMOTE_USER"] ? 201 : 200)
293
331
  r(status, "")
294
332
  rescue MogileFS::Backend::UnregDomainError,
@@ -40,9 +40,9 @@ module OMGF::Pool
40
40
  pool_use(domain) { |mg| mg.get_uris(key, opts) }
41
41
  end
42
42
 
43
- def mg_size_and_paths(domain, key, opts) # :nodoc:
43
+ def mg_size_and_uris(domain, key, opts) # :nodoc:
44
44
  pool_use(domain) do |mg|
45
- [ mg.size(key), mg.get_paths(key, opts) ]
45
+ [ mg.size(key), mg.get_uris(key, opts) ]
46
46
  end
47
47
  end
48
48
  end
@@ -0,0 +1,241 @@
1
+ # -*- encoding: binary -*-
2
+ # :stopdoc:
3
+ # Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
4
+ # License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
5
+ # :startdoc:
6
+ require "thread"
7
+ require "omgf"
8
+ require "kcar"
9
+ require "kgio"
10
+
11
+ # This module makes HEAD requests with reusable HTTP connections to
12
+ # verify paths. This is faster than having the mogilefsd tracker
13
+ # verifying paths, and the client could have broken routing to some
14
+ # the storage nodes the mogilefsd tracker/monitor can see.
15
+ class OMGF::VerifyPaths
16
+
17
+ # private class
18
+ class HeadSock < Kgio::Socket # :nodoc:
19
+ VERSION = '1.0.0'
20
+
21
+ attr_reader :uri
22
+ attr_writer :retry_ok
23
+
24
+ def self.start(uri)
25
+ super(Socket.pack_sockaddr_in(uri.port, uri.host))
26
+ end
27
+
28
+ # returns true if the HTTP connection is reusable
29
+ def http_reusable?
30
+ rv = @kcar.keepalive?
31
+ @kcar.reset
32
+ @headers.clear
33
+ @buf.clear
34
+ @uri = nil
35
+ rv ? setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 1) : close
36
+ rv
37
+ end
38
+
39
+ def http_init(uri, retry_ok = true)
40
+ @uri = uri
41
+ unless defined?(@kcar)
42
+ @kcar = Kcar::Parser.new
43
+ @headers = {}
44
+ @buf = ""
45
+ end
46
+ @retry_ok = retry_ok
47
+
48
+ # prepare the HTTP request
49
+ @req = "HEAD #{@uri.request_uri} HTTP/1.1\r\n" \
50
+ "User-Agent: #{self.class}/#{VERSION}\r\n" \
51
+ "Host: #{@uri.host}:#{@uri.port}\r\n" \
52
+ "\r\n"
53
+ end
54
+
55
+ # returns an array result if successful
56
+ # returns :wait_readable or :wait_writable if incomplete
57
+ # returns a subclass of Exception on errors
58
+ # returns nil on premature EOF (twice)
59
+ def poll_iter(pollset)
60
+ case rv = kgio_trywrite(@req)
61
+ when nil, # done writing, start reading
62
+ String # partial write
63
+ @req = rv # continue looping
64
+ when Symbol
65
+ return pollset[self] = rv # busy
66
+ end while @req
67
+
68
+ case rv = kgio_tryread(666)
69
+ when String
70
+ if ary = @kcar.headers(@headers, @buf << rv)
71
+ pollset.delete(self)
72
+ return ary # success
73
+ end
74
+ # continue looping if incomplete
75
+ when Symbol
76
+ return pollset[self] = rv # busy
77
+ when nil # EOF (premature)
78
+ return maybe_retry(nil, pollset)
79
+ end while true
80
+ rescue => err
81
+ maybe_retry(err, pollset)
82
+ end
83
+
84
+ # returns the err object if we've already retried, nil otherwise
85
+ def maybe_retry(err, pollset)
86
+ pollset.delete(self)
87
+ return err unless @retry_ok
88
+
89
+ # always start a fresh connection on socket errors
90
+ sock = self.class.start(@uri)
91
+ sock.http_init(@uri, false)
92
+ pollset[sock] = :wait_writable
93
+ end
94
+ end
95
+
96
+ def initialize(logger)
97
+ @pool = Hash.new { |hash,host_port| hash[host_port] = [] }
98
+ @logger = logger
99
+ @finishq = Queue.new
100
+ @finisher = nil
101
+ @lock = Mutex.new
102
+ @pid = $$
103
+ end
104
+
105
+ def error(msg)
106
+ @logger.error(msg) if @logger
107
+ end
108
+
109
+ def iter_check(ok, sock, pollset)
110
+ rv = sock.poll_iter(pollset)
111
+ case rv
112
+ when Symbol # in progress
113
+ when Array
114
+ code = rv[0].to_i
115
+ if 200 == code
116
+ ok << sock.uri
117
+ elsif code >= 100 && code <= 999
118
+ error("HEAD #{sock.uri} returned HTTP code: #{code}")
119
+ else
120
+ error("HEAD #{sock.uri} returned #{rv.inspect} (kcar bug?)")
121
+ end
122
+ sock_put(sock)
123
+ when nil # premature EOF
124
+ error("HEAD #{sock.uri} hit socket EOF")
125
+ else
126
+ # exception or some other error return value...
127
+ if rv.respond_to?(:message)
128
+ error("HEAD #{sock.uri} error: #{rv.message} (#{rv.class})")
129
+ else
130
+ error("HEAD #{sock.uri} error (#{rv.class}): #{rv.inspect}")
131
+ end
132
+ end
133
+ rv
134
+ end
135
+
136
+ # this runs in a background thread to cleanup all the requests
137
+ # that didn't finish quickly enough
138
+ def finisher(timeout = 10000)
139
+ begin
140
+ pollset = @finishq.pop # park here when idle
141
+
142
+ while ready = Kgio.poll(pollset.dup, timeout)
143
+ ready.each_key do |sock|
144
+ sock.retry_ok = false
145
+ # try to return good sockets back to the pool
146
+ iter_check([], sock, pollset)
147
+ end
148
+
149
+ # try to stuff the pollset as much as possible for further looping
150
+ while more = (@finishq.pop(true) rescue nil)
151
+ pollset.merge!(more)
152
+ end
153
+ end
154
+
155
+ # connections timed out, kill them
156
+ pollset.each_key { |sock| sock.close }
157
+ rescue => err
158
+ error("#{err.message} (#{err.class})")
159
+ end while true
160
+ end
161
+
162
+ # reorders URIs based on response time
163
+ # This is the main method of this class
164
+ def verify(uris, count, timeout)
165
+ tout = (timeout * 1000).to_i
166
+ pollset = {}
167
+ ok = []
168
+
169
+ uris.each do |uri|
170
+ sock = sock_get(uri) and iter_check(ok, sock, pollset)
171
+ end
172
+
173
+ while ok.size < count && tout > 0 && ! pollset.empty?
174
+ t0 = Time.now
175
+ ready = Kgio.poll(pollset.dup, tout) or break
176
+ tout -= ((Time.now - t0) * 1000).to_i
177
+
178
+ ready.each_key do |sock|
179
+ iter_check(ok, sock, pollset)
180
+ end
181
+ end
182
+
183
+ finish(pollset) unless pollset.empty?
184
+ [ok, uris - ok] # good URLs first
185
+ end
186
+
187
+ # recover any unfinished URLs in pollset asynchronously in the background
188
+ def finish(pollset) # :nodoc:
189
+ @finishq.push(pollset)
190
+ @lock.synchronize do
191
+ unless @finisher && @finisher.alive?
192
+ @finisher = Thread.new { finisher }
193
+ end
194
+ end
195
+ end
196
+
197
+ # returns a string key for the connection pool
198
+ def key_for(uri)
199
+ "#{uri.host}:#{uri.port}"
200
+ end
201
+
202
+ # initializes a cached connection for +uri+ or creates a new one
203
+ def sock_get(uri)
204
+ key = key_for(uri)
205
+
206
+ # detect forks and prevent sharing of connected sockets across processes
207
+ @lock.synchronize do
208
+ if @pid != $$
209
+ @pid = $$
210
+ @pool.clear
211
+ end
212
+ end
213
+
214
+ while sock = @lock.synchronize { @pool[key].pop }
215
+ begin
216
+ # check if sock is still alive and idle
217
+ # :wait_readable is good here
218
+ break if sock.kgio_tryread(1) == :wait_readable
219
+ rescue
220
+ # ignore socket errors, we'll just give them a new socket
221
+ # socket should've been idle, but it was not (or EOFed on us)
222
+ # give them a new one
223
+ end
224
+ sock.close
225
+ end
226
+
227
+ sock ||= HeadSock.start(uri)
228
+ sock.http_init(uri)
229
+ sock
230
+ rescue
231
+ # we'll return nil on any errors
232
+ end
233
+
234
+ # returns an idle socket to the pool
235
+ def sock_put(sock)
236
+ key = key_for(sock.uri)
237
+ sock.http_reusable? and @lock.synchronize { @pool[key] << sock }
238
+ rescue => err
239
+ error("HTTP reuse check failed: #{err.message} (#{err.class})")
240
+ end
241
+ end
@@ -20,6 +20,8 @@ Gem::Specification.new do |s|
20
20
  s.rdoc_options = rdoc_options
21
21
  s.test_files = Dir['test/test_*.rb']
22
22
  s.add_dependency('rack', ['~> 1.3'])
23
+ s.add_dependency('kgio', ['~> 2.7'])
24
+ s.add_dependency('kcar', ['~> 0.3'])
23
25
  s.add_dependency('mogilefs-client', ['~> 3.1'])
24
26
 
25
27
  s.licenses = %w(AGPLv3+)
@@ -67,9 +67,9 @@ module TestMogileFSIntegration
67
67
  [ status, out, err ]
68
68
  end
69
69
 
70
- def setup_mogilefs(plugins = nil)
70
+ def setup_mogilefs(plugins = nil, mogstored = "mogstored")
71
71
  @test_host = "127.0.0.1"
72
- setup_mogstored
72
+ setup_mogstored(mogstored)
73
73
  @tracker = TCPServer.new(@test_host, 0)
74
74
  @tracker_port = @tracker.addr[1]
75
75
 
@@ -173,7 +173,7 @@ EOF
173
173
  raise "#{uri} failed to appear: #{res.inspect}"
174
174
  end
175
175
 
176
- def setup_mogstored
176
+ def setup_mogstored(mogstored = "mogstored")
177
177
  @docroot = Dir.mktmpdir(["mogfresh", "docroot"])
178
178
  Dir.mkdir("#@docroot/dev1")
179
179
  Dir.mkdir("#@docroot/dev2")
@@ -194,7 +194,7 @@ EOF
194
194
  @mogstored_mgmt.close
195
195
  @mogstored_http.close
196
196
 
197
- x!("mogstored", "--daemon", "--config=#{@mogstored_conf.path}")
197
+ x!(mogstored, "--daemon", "--config=#{@mogstored_conf.path}")
198
198
  wait_for_port @mogstored_mgmt_port
199
199
  wait_for_port @mogstored_http_port
200
200
  end
@@ -4,15 +4,17 @@ require './test/integration'
4
4
  require 'rack/mock'
5
5
  require 'open-uri'
6
6
  require 'omgf/hysterical_raisins'
7
+ require 'digest/md5'
7
8
 
8
9
  class TestHystericalRaisins < Test::Unit::TestCase
9
10
  include TestMogileFSIntegration
10
11
  def setup
11
12
  setup_mogilefs
12
- @app = OMGF::HystericalRaisins.new(:hosts => @hosts)
13
- @req = Rack::MockRequest.new(@app)
14
13
  @err = StringIO.new
15
- @opts = { "rack.logger" => Logger.new(@err) }
14
+ logger = Logger.new(@err)
15
+ @opts = { "rack.logger" => logger }
16
+ @app = OMGF::HystericalRaisins.new(:hosts => @hosts, :logger => logger)
17
+ @req = Rack::MockRequest.new(@app)
16
18
  @admin.create_domain("testdom")
17
19
  end
18
20
 
@@ -186,13 +188,21 @@ class TestHystericalRaisins < Test::Unit::TestCase
186
188
 
187
189
  # wait for replication
188
190
  if ENV["EXPENSIVE"]
189
- 200.times do
190
- resp["X-URL-1"] and break
191
- sleep 0.1
191
+ 50.times do
192
192
  resp = @req.head("/testdom/key", @opts)
193
+ assert_equal 200, resp.status, resp.inspect
194
+ resp["X-URL-1"] and break
195
+ sleep 0.5
193
196
  end
194
- assert_kind_of String, resp["X-URL-1"]
197
+ assert_kind_of String, resp["X-URL-1"], resp.inspect
195
198
  assert_equal "BLAH", open(resp["X-URL-1"]).read
199
+
200
+ # Location should not be in X-Alt-Location, too
201
+ resp = @req.get("/testdom/key", @opts)
202
+ assert_equal 302, resp.status, resp.inspect
203
+ assert_kind_of URI, URI(resp["Location"])
204
+ assert_kind_of URI, URI(resp["X-Alt-Location-0"])
205
+ assert_nil resp["X-Alt-Location-1"]
196
206
  end
197
207
 
198
208
  reproxy_test
@@ -232,6 +242,24 @@ class TestHystericalRaisins < Test::Unit::TestCase
232
242
  @app.instance_variable_set(:@reproxy_path, nil)
233
243
  end
234
244
 
245
+ def test_checksums
246
+ mogadm!("class", "add", "testdom", "md5", "--hashtype=MD5")
247
+ 100.times do
248
+ @admin.get_domains["testdom"]["md5"] and break
249
+ sleep 0.5
250
+ end
251
+ assert_equal "MD5", @admin.get_domains["testdom"]["md5"]["hashtype"]
252
+ sio = StringIO.new("HELLO")
253
+ md5 = [ Digest::MD5.digest(sio.string) ].pack('m0')
254
+ opts = @opts.merge(input: sio, method: "PUT", "HTTP_CONTENT_MD5" => md5)
255
+ resp = @req.put("/testdom/cc?class=md5", opts)
256
+ assert_equal 200, resp.status
257
+ @mg = MogileFS::MogileFS.new(hosts: @hosts, domain: "testdom")
258
+ info = @mg.file_info("cc")
259
+ assert_equal "md5", info["class"]
260
+ assert_equal "MD5:#{Digest::MD5.hexdigest(sio.string)}", info["checksum"]
261
+ end
262
+
235
263
  def teardown
236
264
  teardown_mogilefs
237
265
  end
@@ -0,0 +1,54 @@
1
+ # Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
2
+ # License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
3
+ require './test/integration'
4
+ require 'rack/mock'
5
+ require 'open-uri'
6
+ require 'omgf/hysterical_raisins'
7
+ require 'digest/md5'
8
+
9
+ class TestHystericalRaisinsCmogstored < Test::Unit::TestCase
10
+ include TestMogileFSIntegration
11
+ def setup
12
+ setup_mogilefs(nil, "cmogstored")
13
+ @err = StringIO.new
14
+ logger = Logger.new(@err)
15
+ @opts = { "rack.logger" => logger }
16
+ @app = OMGF::HystericalRaisins.new(:hosts => @hosts, :logger => logger)
17
+ @req = Rack::MockRequest.new(@app)
18
+ @admin.create_domain("testdom")
19
+ end
20
+
21
+ def test_checksums_trailer
22
+ mogadm!("class", "add", "testdom", "md5", "--hashtype=MD5")
23
+ 100.times do
24
+ @admin.get_domains["testdom"]["md5"] and break
25
+ sleep 0.5
26
+ end
27
+ assert_equal "MD5", @admin.get_domains["testdom"]["md5"]["hashtype"]
28
+
29
+ sio = StringIO.new("HELLO")
30
+ def sio.read(*_)
31
+ rv = super
32
+ if rv == nil
33
+ md5 = [ Digest::MD5.digest(string) ].pack('m0')
34
+ @rack_env["HTTP_CONTENT_MD5"] = md5
35
+ end
36
+ rv
37
+ end
38
+
39
+ opts = @opts.merge(input: sio, method: "PUT")
40
+ opts["HTTP_TRAILER"] = "Content-MD5"
41
+ env = @req.class.env_for("/testdom/cc?class=md5", opts)
42
+ sio.instance_variable_set(:@rack_env, env)
43
+ status, _, _ = @app.call(env)
44
+ assert_equal 200, status, @err.string
45
+ @mg = MogileFS::MogileFS.new(hosts: @hosts, domain: "testdom")
46
+ info = @mg.file_info("cc")
47
+ assert_equal "md5", info["class"]
48
+ assert_equal "MD5:#{Digest::MD5.hexdigest(sio.string)}", info["checksum"]
49
+ end
50
+
51
+ def teardown
52
+ teardown_mogilefs
53
+ end
54
+ end if `which cmogstored 2>/dev/null` =~ /\bcmogstored\b/
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: !binary |-
3
3
  b21nZg==
4
4
  version: !ruby/object:Gem::Version
5
- version: 0.0.0.GIT
5
+ version: 0.0.1.GIT
6
6
  prerelease: 6
7
7
  platform: ruby
8
8
  authors:
@@ -11,7 +11,7 @@ authors:
11
11
  autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2012-06-20 00:00:00.000000000 Z
14
+ date: 2012-06-23 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: !binary |-
@@ -34,6 +34,48 @@ dependencies:
34
34
  - !ruby/object:Gem::Version
35
35
  version: !binary |-
36
36
  MS4z
37
+ - !ruby/object:Gem::Dependency
38
+ name: !binary |-
39
+ a2dpbw==
40
+ requirement: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - !binary |-
44
+ fj4=
45
+ - !ruby/object:Gem::Version
46
+ version: !binary |-
47
+ Mi43
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ none: false
52
+ requirements:
53
+ - - !binary |-
54
+ fj4=
55
+ - !ruby/object:Gem::Version
56
+ version: !binary |-
57
+ Mi43
58
+ - !ruby/object:Gem::Dependency
59
+ name: !binary |-
60
+ a2Nhcg==
61
+ requirement: !ruby/object:Gem::Requirement
62
+ none: false
63
+ requirements:
64
+ - - !binary |-
65
+ fj4=
66
+ - !ruby/object:Gem::Version
67
+ version: !binary |-
68
+ MC4z
69
+ type: :runtime
70
+ prerelease: false
71
+ version_requirements: !ruby/object:Gem::Requirement
72
+ none: false
73
+ requirements:
74
+ - - !binary |-
75
+ fj4=
76
+ - !ruby/object:Gem::Version
77
+ version: !binary |-
78
+ MC4z
37
79
  - !ruby/object:Gem::Dependency
38
80
  name: !binary |-
39
81
  bW9naWxlZnMtY2xpZW50
@@ -71,6 +113,8 @@ extra_rdoc_files:
71
113
  - lib/omgf.rb
72
114
  - lib/omgf/hysterical_raisins.rb
73
115
  - lib/omgf/pool.rb
116
+ - lib/omgf/verify_paths.rb
117
+ - lib/omgf/version.rb
74
118
  - NEWS
75
119
  - README
76
120
  files:
@@ -88,14 +132,18 @@ files:
88
132
  - README
89
133
  - examples/hyst.README
90
134
  - examples/hyst.bash
135
+ - examples/verify_paths_manual_test.rb
91
136
  - lib/omgf.rb
92
137
  - lib/omgf/hysterical_raisins.rb
93
138
  - lib/omgf/pool.rb
139
+ - lib/omgf/verify_paths.rb
140
+ - lib/omgf/version.rb
94
141
  - omgf.gemspec
95
142
  - pkg.mk
96
143
  - test/integration.rb
97
144
  - test/test_hyst.rb
98
145
  - test/test_hysterical_raisins.rb
146
+ - test/test_hysterical_raisins_cmogstored.rb
99
147
  homepage: http://bogomips.org/omgf/
100
148
  licenses:
101
149
  - !binary |-
@@ -127,6 +175,8 @@ signing_key:
127
175
  specification_version: 3
128
176
  summary: hysterical REST API for MogileFS using Rack
129
177
  test_files:
178
+ - !binary |-
179
+ dGVzdC90ZXN0X2h5c3RlcmljYWxfcmFpc2luc19jbW9nc3RvcmVkLnJi
130
180
  - !binary |-
131
181
  dGVzdC90ZXN0X2h5c3QucmI=
132
182
  - !binary |-