perfectqueue 0.7.32 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. data/.gitignore +6 -0
  2. data/ChangeLog +0 -62
  3. data/Gemfile +3 -0
  4. data/README.md +239 -0
  5. data/Rakefile +19 -0
  6. data/lib/perfectqueue.rb +68 -4
  7. data/lib/perfectqueue/application.rb +30 -0
  8. data/lib/perfectqueue/application/base.rb +27 -0
  9. data/lib/perfectqueue/application/dispatch.rb +73 -0
  10. data/lib/perfectqueue/application/router.rb +69 -0
  11. data/lib/perfectqueue/backend.rb +44 -47
  12. data/lib/perfectqueue/backend/rdb_compat.rb +298 -0
  13. data/lib/perfectqueue/blocking_flag.rb +84 -0
  14. data/lib/perfectqueue/client.rb +117 -0
  15. data/lib/perfectqueue/command/perfectqueue.rb +108 -323
  16. data/lib/perfectqueue/daemons_logger.rb +80 -0
  17. data/lib/perfectqueue/engine.rb +85 -123
  18. data/lib/perfectqueue/error.rb +53 -0
  19. data/lib/perfectqueue/model.rb +37 -0
  20. data/lib/perfectqueue/multiprocess.rb +31 -0
  21. data/lib/perfectqueue/multiprocess/child_process.rb +108 -0
  22. data/lib/perfectqueue/multiprocess/child_process_monitor.rb +109 -0
  23. data/lib/perfectqueue/multiprocess/fork_processor.rb +164 -0
  24. data/lib/perfectqueue/multiprocess/thread_processor.rb +123 -0
  25. data/lib/perfectqueue/queue.rb +58 -0
  26. data/lib/perfectqueue/runner.rb +39 -0
  27. data/lib/perfectqueue/signal_queue.rb +112 -0
  28. data/lib/perfectqueue/task.rb +103 -0
  29. data/lib/perfectqueue/task_metadata.rb +98 -0
  30. data/lib/perfectqueue/task_monitor.rb +189 -0
  31. data/lib/perfectqueue/task_status.rb +27 -0
  32. data/lib/perfectqueue/version.rb +1 -3
  33. data/lib/perfectqueue/worker.rb +114 -196
  34. data/perfectqueue.gemspec +24 -0
  35. data/spec/queue_spec.rb +234 -0
  36. data/spec/spec_helper.rb +44 -0
  37. data/spec/worker_spec.rb +81 -0
  38. metadata +93 -40
  39. checksums.yaml +0 -7
  40. data/README.rdoc +0 -224
  41. data/lib/perfectqueue/backend/null.rb +0 -33
  42. data/lib/perfectqueue/backend/rdb.rb +0 -181
  43. data/lib/perfectqueue/backend/simpledb.rb +0 -139
  44. data/test/backend_test.rb +0 -259
  45. data/test/cat.sh +0 -2
  46. data/test/echo.sh +0 -4
  47. data/test/exec_test.rb +0 -61
  48. data/test/fail.sh +0 -2
  49. data/test/huge.sh +0 -2
  50. data/test/stress.rb +0 -99
  51. data/test/success.sh +0 -2
  52. data/test/test_helper.rb +0 -19
data/test/cat.sh DELETED
@@ -1,2 +0,0 @@
1
- #!/bin/sh
2
- cat > "$1"
data/test/echo.sh DELETED
@@ -1,4 +0,0 @@
1
- #!/bin/sh
2
- out="$1"
3
- shift
4
- echo "$@" > "$out"
data/test/exec_test.rb DELETED
@@ -1,61 +0,0 @@
1
- require File.dirname(__FILE__)+'/test_helper'
2
-
3
- class ExecTest < Test::Unit::TestCase
4
- it 'success' do
5
- success_sh = File.expand_path File.dirname(__FILE__)+"/success.sh"
6
-
7
- task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
8
- e = PerfectQueue::ExecRunner.new(success_sh, task)
9
-
10
- assert_nothing_raised do
11
- e.run
12
- end
13
- end
14
-
15
- it 'fail' do
16
- fail_sh = File.expand_path File.dirname(__FILE__)+"/fail.sh"
17
-
18
- task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
19
- e = PerfectQueue::ExecRunner.new(fail_sh, task)
20
-
21
- assert_raise(RuntimeError) do
22
- e.run
23
- end
24
- end
25
-
26
- it 'stdin' do
27
- cat_sh = File.expand_path File.dirname(__FILE__)+"/cat.sh"
28
- out_tmp = File.expand_path File.dirname(__FILE__)+"/cat.sh.tmp"
29
-
30
- task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
31
- e = PerfectQueue::ExecRunner.new("#{cat_sh} #{out_tmp}", task)
32
-
33
- e.run
34
-
35
- assert_equal 'data1', File.read(out_tmp)
36
- end
37
-
38
- it 'echo' do
39
- echo_sh = File.expand_path File.dirname(__FILE__)+"/echo.sh"
40
- out_tmp = File.expand_path File.dirname(__FILE__)+"/echo.sh.tmp"
41
-
42
- task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
43
- e = PerfectQueue::ExecRunner.new("#{echo_sh} #{out_tmp}", task)
44
-
45
- e.run
46
-
47
- assert_equal "test1\n", File.read(out_tmp)
48
- end
49
-
50
- it 'huge' do
51
- huge_sh = File.expand_path File.dirname(__FILE__)+"/huge.sh"
52
-
53
- task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
54
- e = PerfectQueue::ExecRunner.new(huge_sh, task)
55
-
56
- e.run
57
-
58
- # should finish
59
- end
60
- end
61
-
data/test/fail.sh DELETED
@@ -1,2 +0,0 @@
1
- #!/bin/sh
2
- exit 1
data/test/huge.sh DELETED
@@ -1,2 +0,0 @@
1
- #!/bin/sh
2
- dd if=/dev/zero bs=1024 count=1024
data/test/stress.rb DELETED
@@ -1,99 +0,0 @@
1
- $LOAD_PATH << File.expand_path(File.dirname(__FILE__)+"/../lib")
2
- require 'perfectqueue'
3
- require 'perfectqueue/backend/rdb'
4
- require 'perfectqueue/backend/simpledb'
5
-
6
- class StressTest
7
- def initialize(uri, table, npt, thread)
8
- @db_proc = Proc.new do
9
- PerfectQueue::RDBBackend.new(uri, table)
10
- end
11
- @db_proc.call.create_tables
12
- @npt = npt
13
- @thread = thread
14
- end
15
-
16
- class ThreadMain < Thread
17
- def initialize(key_prefix, db, num)
18
- @key_prefix = key_prefix
19
- @db = db
20
- @num = num
21
- super(&method(:run))
22
- end
23
-
24
- def run
25
- @num.times {|i|
26
- now = Time.now.to_i
27
- @db.submit("#{@key_prefix}-#{i}", "data", now)
28
- token, task = @db.acquire(now+60, now)
29
- if token == nil
30
- puts "acquire failed"
31
- next
32
- end
33
- @db.update(token, now+70)
34
- @db.finish(token, now+80)
35
- }
36
- end
37
- end
38
-
39
- def run
40
- threads = []
41
- key_prefix = "stress-#{'%08x'%rand(2**32)}"
42
- now = Time.now
43
- @thread.times {|i|
44
- threads << ThreadMain.new("#{key_prefix}-#{i}", @db_proc.call, @npt)
45
- }
46
- threads.each {|t|
47
- t.join
48
- }
49
- finish = Time.now
50
-
51
- elapsed = finish - now
52
- puts "#{elapsed} sec."
53
- puts "#{@npt * @thread / elapsed} req/sec."
54
- puts "#{elapsed / (@npt * @thread)} sec/req."
55
- end
56
- end
57
-
58
- require 'optparse'
59
-
60
- op = OptionParser.new
61
- op.banner += " <uri> <table>"
62
-
63
- num = 100
64
- thread = 1
65
-
66
- op.on('-n', '--num N', Integer) {|n|
67
- num = n
68
- }
69
- op.on('-t', '--thread N', Integer) {|n|
70
- thread = n
71
- }
72
-
73
- begin
74
- op.parse!(ARGV)
75
-
76
- if ARGV.length != 2
77
- puts op.to_s
78
- exit 1
79
- end
80
-
81
- uri = ARGV[0]
82
- table = ARGV[1]
83
-
84
- rescue
85
- puts op.to_s
86
- puts $!
87
- exit 1
88
- end
89
-
90
- npt = num / thread
91
- num = npt * thread
92
-
93
- puts "num: #{num}"
94
- puts "thread: #{thread}"
95
- puts "num/thread: #{npt}"
96
-
97
- t = StressTest.new(uri, table, npt, thread)
98
- t.run
99
-
data/test/success.sh DELETED
@@ -1,2 +0,0 @@
1
- #!/bin/sh
2
- exit 0
data/test/test_helper.rb DELETED
@@ -1,19 +0,0 @@
1
- require 'simplecov'
2
- SimpleCov.start
3
- require 'test/unit'
4
- $LOAD_PATH << File.dirname(__FILE__)+"/../lib"
5
- require 'perfectqueue'
6
- require 'shellwords'
7
- require 'perfectqueue/backend/rdb'
8
- require 'perfectqueue/backend/simpledb'
9
- require 'fileutils'
10
-
11
- class Test::Unit::TestCase
12
- #class << self
13
- # alias_method :it, :test
14
- #end
15
- def self.it(name, &block)
16
- define_method("test_#{name}", &block)
17
- end
18
- end
19
-