queueing_proxy 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +2 -0
- data/.rvmrc +47 -0
- data/Gemfile.lock +28 -0
- data/Rakefile +1 -2
- data/bin/queueing_proxy +29 -1
- data/bin/upstream +19 -0
- data/lib/queueing_proxy.rb +3 -6
- data/lib/queueing_proxy/dsl.rb +52 -22
- data/lib/queueing_proxy/frontend.rb +71 -0
- data/lib/queueing_proxy/version.rb +1 -1
- data/lib/queueing_proxy/worker.rb +126 -0
- data/queueing_proxy.gemspec +2 -4
- metadata +58 -103
- data/lib/queueing_proxy/cli.rb +0 -17
- data/lib/queueing_proxy/dispatcher.rb +0 -60
- data/lib/queueing_proxy/queuer.rb +0 -57
data/.gitignore
ADDED
data/.rvmrc
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
# This is an RVM Project .rvmrc file, used to automatically load the ruby
|
4
|
+
# development environment upon cd'ing into the directory
|
5
|
+
|
6
|
+
# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional.
|
7
|
+
environment_id="ruby-1.9.2-p290@queueing_proxy"
|
8
|
+
|
9
|
+
#
|
10
|
+
# Uncomment following line if you want options to be set only for given project.
|
11
|
+
#
|
12
|
+
# PROJECT_JRUBY_OPTS=( --1.9 )
|
13
|
+
|
14
|
+
#
|
15
|
+
# First we attempt to load the desired environment directly from the environment
|
16
|
+
# file. This is very fast and efficient compared to running through the entire
|
17
|
+
# CLI and selector. If you want feedback on which environment was used then
|
18
|
+
# insert the word 'use' after --create as this triggers verbose mode.
|
19
|
+
#
|
20
|
+
if [[ -d "${rvm_path:-$HOME/.rvm}/environments" \
|
21
|
+
&& -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
|
22
|
+
then
|
23
|
+
\. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
|
24
|
+
|
25
|
+
if [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]]
|
26
|
+
then
|
27
|
+
. "${rvm_path:-$HOME/.rvm}/hooks/after_use"
|
28
|
+
fi
|
29
|
+
else
|
30
|
+
# If the environment file has not yet been created, use the RVM CLI to select.
|
31
|
+
if ! rvm --create "$environment_id"
|
32
|
+
then
|
33
|
+
echo "Failed to create RVM environment '${environment_id}'."
|
34
|
+
exit 1
|
35
|
+
fi
|
36
|
+
fi
|
37
|
+
|
38
|
+
#
|
39
|
+
# If you use an RVM gemset file to install a list of gems (*.gems), you can have
|
40
|
+
# it be automatically loaded. Uncomment the following and adjust the filename if
|
41
|
+
# necessary.
|
42
|
+
#
|
43
|
+
# filename=".gems"
|
44
|
+
# if [[ -s "$filename" ]] ; then
|
45
|
+
# rvm gemset import "$filename" | grep -v already | grep -v listed | grep -v complete | sed '/^$/d'
|
46
|
+
# fi
|
47
|
+
|
data/Gemfile.lock
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
queueing_proxy (0.0.4)
|
5
|
+
em-jack
|
6
|
+
eventmachine
|
7
|
+
http_parser.rb
|
8
|
+
thin
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: http://rubygems.org/
|
12
|
+
specs:
|
13
|
+
daemons (1.1.4)
|
14
|
+
em-jack (0.1.3)
|
15
|
+
eventmachine
|
16
|
+
eventmachine (0.12.10)
|
17
|
+
http_parser.rb (0.5.2)
|
18
|
+
rack (1.3.2)
|
19
|
+
thin (1.2.11)
|
20
|
+
daemons (>= 1.0.9)
|
21
|
+
eventmachine (>= 0.12.6)
|
22
|
+
rack (>= 1.0.0)
|
23
|
+
|
24
|
+
PLATFORMS
|
25
|
+
ruby
|
26
|
+
|
27
|
+
DEPENDENCIES
|
28
|
+
queueing_proxy!
|
data/Rakefile
CHANGED
@@ -1,2 +1 @@
|
|
1
|
-
require '
|
2
|
-
Bundler::GemHelper.install_tasks
|
1
|
+
require 'bundler/gem_tasks'
|
data/bin/queueing_proxy
CHANGED
@@ -4,5 +4,33 @@ $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
|
|
4
4
|
|
5
5
|
require 'queueing_proxy'
|
6
6
|
|
7
|
-
|
7
|
+
EM.run{
|
8
|
+
# many-to-many routing topology. Whoa!
|
9
|
+
QueueingProxy {
|
10
|
+
logger Logger.new($stdout)
|
8
11
|
|
12
|
+
from '0.0.0.0', 11000
|
13
|
+
from '0.0.0.0', 10007
|
14
|
+
|
15
|
+
to '127.0.0.1', 10001, do
|
16
|
+
queue_with '0.0.0.0', 'http-queue'
|
17
|
+
workers 1
|
18
|
+
end
|
19
|
+
|
20
|
+
to '127.0.0.1', 10009 do
|
21
|
+
queue_with '0.0.0.0', 'message-gateway-queue'
|
22
|
+
workers 1
|
23
|
+
end
|
24
|
+
}.run
|
25
|
+
|
26
|
+
QueueingProxy {
|
27
|
+
logger Logger.new($stdout)
|
28
|
+
|
29
|
+
from '0.0.0.0', 10002
|
30
|
+
|
31
|
+
to('127.0.0.1', 10001) do
|
32
|
+
workers 1
|
33
|
+
queue_with('0.0.0.0', 'http-queue2')
|
34
|
+
end
|
35
|
+
}.run
|
36
|
+
}
|
data/bin/upstream
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require 'rubygems'
|
3
|
+
require 'thin'
|
4
|
+
|
5
|
+
# curl "http://localhost:10001/?wait=5&status=500"
|
6
|
+
# ---
|
7
|
+
# wait - How many seconds should I wait before returning the status?
|
8
|
+
# status - HTTP return code
|
9
|
+
|
10
|
+
Thin::Server.start('127.0.0.1', 10001, Proc.new{|env|
|
11
|
+
req = Rack::Request.new(env)
|
12
|
+
puts "GET /?#{req.query_string}"
|
13
|
+
EventMachine::Timer.new(req.params['wait'].to_i) {
|
14
|
+
response = [(req.params['status'] || 200).to_i, {}, '']
|
15
|
+
env['async.callback'].call response
|
16
|
+
}
|
17
|
+
# Tell thin to chill, we're gonna run this stack async-style
|
18
|
+
[-1, {}, []]
|
19
|
+
})
|
data/lib/queueing_proxy.rb
CHANGED
@@ -3,18 +3,15 @@ require 'eventmachine'
|
|
3
3
|
require 'em-jack'
|
4
4
|
require 'json'
|
5
5
|
|
6
|
-
require 'queueing_proxy/cli'
|
7
|
-
require 'queueing_proxy/dsl'
|
8
|
-
require 'queueing_proxy/queuer'
|
9
6
|
require 'queueing_proxy/version'
|
10
|
-
require 'queueing_proxy/
|
7
|
+
require 'queueing_proxy/dsl'
|
8
|
+
require 'queueing_proxy/frontend'
|
9
|
+
require 'queueing_proxy/worker'
|
11
10
|
|
12
11
|
module QueueingProxy
|
13
|
-
|
14
12
|
def self.from(host, port)
|
15
13
|
DSL.new.from(host, port)
|
16
14
|
end
|
17
|
-
|
18
15
|
end
|
19
16
|
|
20
17
|
def QueueingProxy(&blk)
|
data/lib/queueing_proxy/dsl.rb
CHANGED
@@ -2,32 +2,47 @@ require 'logger'
|
|
2
2
|
|
3
3
|
module QueueingProxy
|
4
4
|
class DSL
|
5
|
+
class Backend
|
6
|
+
attr_reader :tube, :host
|
7
|
+
|
8
|
+
def initialize(host='localhost', port=80, workers=4, &block)
|
9
|
+
@host, @port, @workers, @beanstalk_host, @tube = host, port, workers, 'localhost', 'default'
|
10
|
+
instance_exec(&block) if block_given?
|
11
|
+
self
|
12
|
+
end
|
13
|
+
|
14
|
+
def queue_with(beanstalk, tube)
|
15
|
+
@beanstalk, @tube = beanstalk, tube
|
16
|
+
self
|
17
|
+
end
|
18
|
+
|
19
|
+
def workers(workers=nil)
|
20
|
+
if workers
|
21
|
+
@workers = workers
|
22
|
+
self
|
23
|
+
else
|
24
|
+
(1..@workers).map { Worker.new(@logger, @host, @port, @beanstalk_host, @tube) }
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def logger(logger)
|
29
|
+
@logger = logger
|
30
|
+
self
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
5
34
|
def initialize(&block)
|
6
|
-
@
|
7
|
-
@logger = Logger.new(STDOUT)
|
35
|
+
@logger = Logger.new($stdout)
|
8
36
|
instance_eval(&block) if block
|
9
37
|
end
|
10
38
|
|
11
|
-
def to(host, port)
|
12
|
-
|
13
|
-
@to_port = port
|
14
|
-
self
|
15
|
-
end
|
16
|
-
|
17
|
-
def from(host, port)
|
18
|
-
@from_host = host
|
19
|
-
@from_port = port
|
20
|
-
self
|
21
|
-
end
|
22
|
-
|
23
|
-
def times(count)
|
24
|
-
@dispatcher_count = count
|
39
|
+
def to(host, port=80, &block)
|
40
|
+
backends << Backend.new(host, port, &block)
|
25
41
|
self
|
26
42
|
end
|
27
43
|
|
28
|
-
def
|
29
|
-
|
30
|
-
@tube = tube
|
44
|
+
def from(host, port=80, &block)
|
45
|
+
frontends << [host, port]
|
31
46
|
self
|
32
47
|
end
|
33
48
|
|
@@ -35,14 +50,29 @@ module QueueingProxy
|
|
35
50
|
@logger = logger
|
36
51
|
self
|
37
52
|
end
|
38
|
-
|
53
|
+
|
39
54
|
def run
|
40
55
|
unless EM.reactor_running?
|
41
56
|
EM.run{ run }
|
42
57
|
else
|
43
|
-
|
44
|
-
|
58
|
+
frontends.each do |host, port|
|
59
|
+
Frontend.new(@logger, host, port, backends).run
|
60
|
+
end
|
61
|
+
# Setup multiple backends
|
62
|
+
backends.each {|b|
|
63
|
+
b.logger @logger
|
64
|
+
b.workers.each(&:run)
|
65
|
+
}
|
45
66
|
end
|
46
67
|
end
|
68
|
+
|
69
|
+
private
|
70
|
+
def frontends
|
71
|
+
@frontends ||= []
|
72
|
+
end
|
73
|
+
|
74
|
+
def backends
|
75
|
+
@backends ||= []
|
76
|
+
end
|
47
77
|
end
|
48
78
|
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
module QueueingProxy
|
2
|
+
class Frontend
|
3
|
+
attr_reader :logger
|
4
|
+
|
5
|
+
def initialize(logger, host, port, backends)
|
6
|
+
@logger, @host, @port, @backends = logger, host, port, backends
|
7
|
+
logger.info "Starting queuer on #{host}:#{port} using beanstalk at #{backends.map{|b| "#{b.tube}@#{b.host}"}.join(', ')}"
|
8
|
+
end
|
9
|
+
|
10
|
+
def beanstalks
|
11
|
+
@beanstalks ||= @backends.map {|b|
|
12
|
+
connection = EMJack::Connection.new(:host => b.host, :tube => b.tube)
|
13
|
+
connection.errback { logger.error "Couldn't connect to beanstalk" }
|
14
|
+
connection
|
15
|
+
}
|
16
|
+
end
|
17
|
+
|
18
|
+
def run
|
19
|
+
app = proc do |env|
|
20
|
+
logger.info "Frontend frontend #{env['HTTP_VERSION']} #{env['PATH_INFO']} #{env['REQUEST_METHOD']}"
|
21
|
+
[204, {}, []]
|
22
|
+
end
|
23
|
+
backend = FakeBackend.new
|
24
|
+
|
25
|
+
EM.start_server(@host, @port, Queuer) do |conn|
|
26
|
+
conn.beanstalks = beanstalks
|
27
|
+
conn.app = app
|
28
|
+
conn.backend = backend
|
29
|
+
conn.logger = logger
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# We need this to make Thin happy
|
34
|
+
class FakeBackend
|
35
|
+
def connection_finished(conn)
|
36
|
+
end
|
37
|
+
|
38
|
+
def ssl?
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
class Queuer < Thin::Connection
|
43
|
+
attr_accessor :beanstalks, :logger
|
44
|
+
|
45
|
+
def post_init
|
46
|
+
@data = ''
|
47
|
+
super
|
48
|
+
end
|
49
|
+
|
50
|
+
def receive_data(data)
|
51
|
+
@data << data
|
52
|
+
super(data)
|
53
|
+
end
|
54
|
+
|
55
|
+
def unbind
|
56
|
+
queue_data
|
57
|
+
super
|
58
|
+
end
|
59
|
+
|
60
|
+
def queue_data
|
61
|
+
if @data != ''
|
62
|
+
beanstalks.each {|b|
|
63
|
+
b.put(@data) {|id|
|
64
|
+
logger.info "Frontend enqueued job #{id} to #{b}"
|
65
|
+
}
|
66
|
+
}
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,126 @@
|
|
1
|
+
require "http/parser"
|
2
|
+
|
3
|
+
module QueueingProxy
|
4
|
+
class Worker
|
5
|
+
# 4294967295 is from the Beanstalkd protocol as the least important
|
6
|
+
# possible job priority. 0 is the highest pri.
|
7
|
+
module Priority
|
8
|
+
Lowest = 4294967295
|
9
|
+
Highest = 0
|
10
|
+
end
|
11
|
+
|
12
|
+
attr_reader :logger
|
13
|
+
|
14
|
+
def initialize(logger, to_host, to_port, beanstalk_host, tube, retries=3)
|
15
|
+
@logger, @to_host, @to_port, @beanstalk_host, @tube, @retries = logger, to_host, to_port, beanstalk_host, tube, retries
|
16
|
+
end
|
17
|
+
|
18
|
+
# Setup a beanstalk connection
|
19
|
+
def beanstalk
|
20
|
+
@beanstalk ||= EMJack::Connection.new(:host => @beanstalk_host, :tube => @tube)
|
21
|
+
end
|
22
|
+
|
23
|
+
# Run the beanstalk consumer that pops the job off the queue and passes it
|
24
|
+
# to a connection object that makes an upstream connection
|
25
|
+
def run
|
26
|
+
logger.debug "Worker #{object_id} reporting again for duty"
|
27
|
+
beanstalk.reserve {|job|
|
28
|
+
logger.info "Worker #{object_id} reserved #{job}"
|
29
|
+
upstream = Upstream.new(job.body, @to_host, @to_port, 15, logger).request
|
30
|
+
upstream.errback{
|
31
|
+
logger.info "Worker #{object_id} upstream connection timed-out with #{job}."
|
32
|
+
# If there's an upstream problem, try this a fe more times, then bury it
|
33
|
+
job.stats{|stats|
|
34
|
+
if stats['reserves'] < @retries
|
35
|
+
logger.info "Worker #{object_id} delayed for 5 seconds."
|
36
|
+
job.release(:delay => 5){ run }
|
37
|
+
else
|
38
|
+
logger.info "Worker #{object_id} max #{@retries} retries. Burying."
|
39
|
+
job.bury(Priority::Lowest) { run }
|
40
|
+
end
|
41
|
+
}
|
42
|
+
}
|
43
|
+
upstream.callback {
|
44
|
+
case status = Integer(upstream.response.status_code)
|
45
|
+
when 200..299
|
46
|
+
logger.info "Worker #{object_id}. Deleting #{job.jobid}."
|
47
|
+
job.delete { run}
|
48
|
+
else
|
49
|
+
logger.info "Worker #{object_id}. Burying #{job} for inspection."
|
50
|
+
job.bury(Priority::Lowest) { run }
|
51
|
+
end
|
52
|
+
}
|
53
|
+
}.errback{|status|
|
54
|
+
case status
|
55
|
+
when :disconnected
|
56
|
+
logger.error "Worker #{object_id} reservation error. Rescheduling."
|
57
|
+
else
|
58
|
+
logger.error "Worker #{object_id} unhandled error #{status}."
|
59
|
+
end
|
60
|
+
run # Keep on chuggin partner!
|
61
|
+
}
|
62
|
+
end
|
63
|
+
|
64
|
+
# Defferable upstream connection
|
65
|
+
class Upstream
|
66
|
+
include EventMachine::Deferrable
|
67
|
+
|
68
|
+
# The connection handler for EM.
|
69
|
+
module Client
|
70
|
+
attr_accessor :upstream
|
71
|
+
|
72
|
+
# Succesful connection!
|
73
|
+
def connection_completed
|
74
|
+
# Setup our callback for when the upstream response hits us
|
75
|
+
upstream.response.on_headers_complete = Proc.new {
|
76
|
+
upstream.succeed
|
77
|
+
close_connection
|
78
|
+
:stop
|
79
|
+
}
|
80
|
+
|
81
|
+
# Send the HTTP request upstream
|
82
|
+
send_data upstream.payload
|
83
|
+
end
|
84
|
+
|
85
|
+
# Read the response of the upstream proxy
|
86
|
+
def receive_data(data)
|
87
|
+
begin
|
88
|
+
upstream.response << data
|
89
|
+
rescue HTTP::Parser::Error
|
90
|
+
upstream.fail
|
91
|
+
close_connection
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
# If something bad happens to the connection, bind gets called
|
96
|
+
def unbind
|
97
|
+
upstream.fail
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
attr_accessor :payload, :host, :port, :timeout, :logger
|
102
|
+
attr_reader :response
|
103
|
+
|
104
|
+
def initialize(payload, host, port, timeout, logger)
|
105
|
+
@response = Http::Parser.new
|
106
|
+
@payload, @host, @port, @timeout, @logger = payload, host, port, timeout, logger
|
107
|
+
end
|
108
|
+
|
109
|
+
# Connect to the upstream server and send the HTTP payload
|
110
|
+
def request
|
111
|
+
begin
|
112
|
+
EventMachine.connect(host, port, Client) {|c|
|
113
|
+
c.upstream = self
|
114
|
+
c.comm_inactivity_timeout = timeout
|
115
|
+
c.pending_connect_timeout = timeout
|
116
|
+
}
|
117
|
+
# TODO - Is this async/deferrable?
|
118
|
+
rescue => e
|
119
|
+
logger.error e
|
120
|
+
fail # If something explodes here, fail the defferable
|
121
|
+
end
|
122
|
+
self
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
data/queueing_proxy.gemspec
CHANGED
@@ -24,8 +24,7 @@ Gem::Specification.new do |s|
|
|
24
24
|
s.add_runtime_dependency 'eventmachine'
|
25
25
|
s.add_runtime_dependency 'em-jack'
|
26
26
|
s.add_runtime_dependency 'thin'
|
27
|
-
s.add_runtime_dependency '
|
28
|
-
s.add_runtime_dependency 'thor', '>= 0.13.8'
|
27
|
+
s.add_runtime_dependency 'http_parser.rb'
|
29
28
|
|
30
29
|
if s.respond_to? :specification_version then
|
31
30
|
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
@@ -36,5 +35,4 @@ Gem::Specification.new do |s|
|
|
36
35
|
end
|
37
36
|
else
|
38
37
|
end
|
39
|
-
end
|
40
|
-
|
38
|
+
end
|
metadata
CHANGED
@@ -1,148 +1,103 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: queueing_proxy
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
5
|
-
prerelease:
|
6
|
-
segments:
|
7
|
-
- 0
|
8
|
-
- 0
|
9
|
-
- 4
|
10
|
-
version: 0.0.4
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.5
|
5
|
+
prerelease:
|
11
6
|
platform: ruby
|
12
|
-
authors:
|
7
|
+
authors:
|
13
8
|
- Joshua Hull
|
14
9
|
autorequire:
|
15
10
|
bindir: bin
|
16
11
|
cert_chain: []
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
dependencies:
|
21
|
-
- !ruby/object:Gem::Dependency
|
12
|
+
date: 2010-07-21 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
22
15
|
name: eventmachine
|
23
|
-
|
24
|
-
requirement: &id001 !ruby/object:Gem::Requirement
|
16
|
+
requirement: &70269188984240 !ruby/object:Gem::Requirement
|
25
17
|
none: false
|
26
|
-
requirements:
|
27
|
-
- -
|
28
|
-
- !ruby/object:Gem::Version
|
29
|
-
|
30
|
-
segments:
|
31
|
-
- 0
|
32
|
-
version: "0"
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
33
22
|
type: :runtime
|
34
|
-
version_requirements: *id001
|
35
|
-
- !ruby/object:Gem::Dependency
|
36
|
-
name: em-jack
|
37
23
|
prerelease: false
|
38
|
-
|
24
|
+
version_requirements: *70269188984240
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: em-jack
|
27
|
+
requirement: &70269188983560 !ruby/object:Gem::Requirement
|
39
28
|
none: false
|
40
|
-
requirements:
|
41
|
-
- -
|
42
|
-
- !ruby/object:Gem::Version
|
43
|
-
|
44
|
-
segments:
|
45
|
-
- 0
|
46
|
-
version: "0"
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
47
33
|
type: :runtime
|
48
|
-
version_requirements: *id002
|
49
|
-
- !ruby/object:Gem::Dependency
|
50
|
-
name: thin
|
51
34
|
prerelease: false
|
52
|
-
|
35
|
+
version_requirements: *70269188983560
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: thin
|
38
|
+
requirement: &70269188982800 !ruby/object:Gem::Requirement
|
53
39
|
none: false
|
54
|
-
requirements:
|
55
|
-
- -
|
56
|
-
- !ruby/object:Gem::Version
|
57
|
-
|
58
|
-
segments:
|
59
|
-
- 0
|
60
|
-
version: "0"
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
61
44
|
type: :runtime
|
62
|
-
version_requirements: *id003
|
63
|
-
- !ruby/object:Gem::Dependency
|
64
|
-
name: json
|
65
45
|
prerelease: false
|
66
|
-
|
46
|
+
version_requirements: *70269188982800
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: http_parser.rb
|
49
|
+
requirement: &70269188981260 !ruby/object:Gem::Requirement
|
67
50
|
none: false
|
68
|
-
requirements:
|
69
|
-
- -
|
70
|
-
- !ruby/object:Gem::Version
|
71
|
-
|
72
|
-
segments:
|
73
|
-
- 0
|
74
|
-
version: "0"
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
75
55
|
type: :runtime
|
76
|
-
version_requirements: *id004
|
77
|
-
- !ruby/object:Gem::Dependency
|
78
|
-
name: thor
|
79
56
|
prerelease: false
|
80
|
-
|
81
|
-
none: false
|
82
|
-
requirements:
|
83
|
-
- - ">="
|
84
|
-
- !ruby/object:Gem::Version
|
85
|
-
hash: 59
|
86
|
-
segments:
|
87
|
-
- 0
|
88
|
-
- 13
|
89
|
-
- 8
|
90
|
-
version: 0.13.8
|
91
|
-
type: :runtime
|
92
|
-
version_requirements: *id005
|
57
|
+
version_requirements: *70269188981260
|
93
58
|
description: Queueing proxy backed by EM/Beanstalk for a very weird purpose.
|
94
59
|
email: joshbuddy@gmail.com
|
95
60
|
executables: []
|
96
|
-
|
97
61
|
extensions: []
|
98
|
-
|
99
62
|
extra_rdoc_files: []
|
100
|
-
|
101
|
-
|
63
|
+
files:
|
64
|
+
- .gitignore
|
65
|
+
- .rvmrc
|
102
66
|
- Gemfile
|
67
|
+
- Gemfile.lock
|
103
68
|
- Rakefile
|
104
69
|
- bin/queueing_proxy
|
70
|
+
- bin/upstream
|
105
71
|
- ext/gem_rake.rb
|
106
72
|
- lib/queueing_proxy.rb
|
107
|
-
- lib/queueing_proxy/cli.rb
|
108
|
-
- lib/queueing_proxy/dispatcher.rb
|
109
73
|
- lib/queueing_proxy/dsl.rb
|
110
|
-
- lib/queueing_proxy/
|
74
|
+
- lib/queueing_proxy/frontend.rb
|
111
75
|
- lib/queueing_proxy/version.rb
|
76
|
+
- lib/queueing_proxy/worker.rb
|
112
77
|
- queueing_proxy.gemspec
|
113
|
-
has_rdoc: true
|
114
78
|
homepage: http://github.com/joshbuddy/queueing_proxy
|
115
79
|
licenses: []
|
116
|
-
|
117
80
|
post_install_message:
|
118
|
-
rdoc_options:
|
81
|
+
rdoc_options:
|
119
82
|
- --charset=UTF-8
|
120
|
-
require_paths:
|
83
|
+
require_paths:
|
121
84
|
- lib
|
122
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
85
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
123
86
|
none: false
|
124
|
-
requirements:
|
125
|
-
- -
|
126
|
-
- !ruby/object:Gem::Version
|
127
|
-
|
128
|
-
|
129
|
-
- 0
|
130
|
-
version: "0"
|
131
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ! '>='
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '0'
|
91
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
132
92
|
none: false
|
133
|
-
requirements:
|
134
|
-
- -
|
135
|
-
- !ruby/object:Gem::Version
|
136
|
-
|
137
|
-
segments:
|
138
|
-
- 0
|
139
|
-
version: "0"
|
93
|
+
requirements:
|
94
|
+
- - ! '>='
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
140
97
|
requirements: []
|
141
|
-
|
142
98
|
rubyforge_project: queueing_proxy
|
143
|
-
rubygems_version: 1.
|
99
|
+
rubygems_version: 1.8.10
|
144
100
|
signing_key:
|
145
101
|
specification_version: 3
|
146
102
|
summary: Queueing proxy backed by EM/Beanstalk for a very weird purpose.
|
147
103
|
test_files: []
|
148
|
-
|
data/lib/queueing_proxy/cli.rb
DELETED
@@ -1,60 +0,0 @@
|
|
1
|
-
module QueueingProxy
|
2
|
-
class Dispatcher
|
3
|
-
attr_reader :logger
|
4
|
-
|
5
|
-
def initialize(logger, to_host, to_port, beanstalk_host, tube)
|
6
|
-
@logger, @to_host, @to_port, @beanstalk_host, @tube = logger, to_host, to_port, beanstalk_host, tube
|
7
|
-
@beanstalk = EMJack::Connection.new(:host => @beanstalk_host, :tube => @tube)
|
8
|
-
@beanstalk.watch(@tube)
|
9
|
-
logger.info "Starting dispatcher on #{to_host}:#{to_port} using beanstalk at #{tube}@#{beanstalk_host}"
|
10
|
-
end
|
11
|
-
|
12
|
-
def run
|
13
|
-
@beanstalk.reserve do |job|
|
14
|
-
logger.info "Dispatching #{job.jobid}"
|
15
|
-
parsed_job = JSON.parse(job.body)
|
16
|
-
begin
|
17
|
-
EventMachine.connect(@to_host, @to_port, DispatchClient) { |c|
|
18
|
-
c.payload = parsed_job['data']
|
19
|
-
c.dispatcher = self
|
20
|
-
c.logger = logger
|
21
|
-
c.job = job
|
22
|
-
c.dispatcher = self
|
23
|
-
}
|
24
|
-
rescue EventMachine::ConnectionError
|
25
|
-
job.release(:delay => 5)
|
26
|
-
logger.info("Problem connecting")
|
27
|
-
EM.add_timer(5){ run }
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
class DispatchClient < EventMachine::Connection
|
33
|
-
attr_accessor :payload, :dispatcher, :logger, :dispatcher, :job
|
34
|
-
|
35
|
-
def connection_completed
|
36
|
-
send_data(payload)
|
37
|
-
end
|
38
|
-
|
39
|
-
def receive_data(data)
|
40
|
-
status = Integer(data[/^HTTP\/(1\.1|1\.0) (\d+)/, 2])
|
41
|
-
close_connection
|
42
|
-
case status
|
43
|
-
when 200
|
44
|
-
logger.info "Done dispatching #{job.jobid}"
|
45
|
-
job.delete
|
46
|
-
when 500..599
|
47
|
-
logger.info "Error #{status}"
|
48
|
-
job.release(:delay => 5)
|
49
|
-
else
|
50
|
-
logger.info "Done dispatching #{job.jobid} -- #{status}"
|
51
|
-
job.delete
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
def unbind
|
56
|
-
dispatcher.run
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
@@ -1,57 +0,0 @@
|
|
1
|
-
module QueueingProxy
|
2
|
-
class Queuer
|
3
|
-
attr_reader :logger
|
4
|
-
|
5
|
-
def initialize(logger, host, port, beanstalk_host, tube)
|
6
|
-
@logger, @host, @port, @beanstalk_host, @tube = logger, host, port, beanstalk_host, tube
|
7
|
-
logger.info "Starting queuer on #{host}:#{port} using beanstalk at #{tube}@#{beanstalk_host}"
|
8
|
-
end
|
9
|
-
|
10
|
-
def run
|
11
|
-
beanstalk = EMJack::Connection.new(:host => @beanstalk_host, :tube => @tube)
|
12
|
-
app = proc do |env|
|
13
|
-
logger.info "Queueing #{env['HTTP_VERSION']} #{env['PATH_INFO']} #{env['REQUEST_METHOD']}"
|
14
|
-
[200, {}, []]
|
15
|
-
end
|
16
|
-
backend = FakeBackend.new
|
17
|
-
EM.start_server(@host, @port, QueuerConnection) do |conn|
|
18
|
-
conn.beanstalk = beanstalk
|
19
|
-
conn.app = app
|
20
|
-
conn.backend = backend
|
21
|
-
conn.logger = logger
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
class FakeBackend
|
26
|
-
def connection_finished(conn)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
class QueuerConnection < Thin::Connection
|
31
|
-
attr_accessor :beanstalk, :logger
|
32
|
-
|
33
|
-
def post_init
|
34
|
-
@data = ''
|
35
|
-
super
|
36
|
-
end
|
37
|
-
|
38
|
-
def receive_data(data)
|
39
|
-
@data << data
|
40
|
-
super(data)
|
41
|
-
end
|
42
|
-
|
43
|
-
def unbind
|
44
|
-
queue_data
|
45
|
-
super
|
46
|
-
end
|
47
|
-
|
48
|
-
def queue_data
|
49
|
-
if @data != ''
|
50
|
-
beanstalk.put({:data => @data, :time => Time.new.to_i}.to_json) { |id|
|
51
|
-
logger.info "Job queued #{id}"
|
52
|
-
}
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|