jp 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/jp +10 -0
- data/bin/jp-pool-size +27 -0
- data/bin/jp-stats +86 -0
- data/bin/jp-unlocker +10 -0
- data/lib/rb/jp/client.rb +23 -0
- data/lib/rb/jp/consumer.rb +83 -0
- data/lib/rb/jp/gen-rb/facebook_service.rb +700 -0
- data/lib/rb/jp/gen-rb/fb303_constants.rb +8 -0
- data/lib/rb/jp/gen-rb/fb303_types.rb +18 -0
- data/lib/rb/jp/gen-rb/job_pool.rb +218 -0
- data/lib/rb/jp/gen-rb/job_pool_instrumented.rb +381 -0
- data/lib/rb/jp/gen-rb/jp_constants.rb +10 -0
- data/lib/rb/jp/gen-rb/jp_types.rb +61 -0
- data/lib/rb/jp/producer.rb +45 -0
- data/lib/rb/jp/server/configuration_loader.rb +71 -0
- data/lib/rb/jp/server/handler.rb +94 -0
- data/lib/rb/jp/server/instrumented_server.rb +87 -0
- data/lib/rb/jp/server/mongo_connection.rb +29 -0
- data/lib/rb/jp/server/pools.rb +28 -0
- data/lib/rb/jp/server/server.rb +74 -0
- data/lib/rb/jp/server/unlocker.rb +50 -0
- data/lib/rb/jp/thrift.rb +4 -0
- metadata +113 -0
@@ -0,0 +1,61 @@
|
|
1
|
+
#
|
2
|
+
# Autogenerated by Thrift
|
3
|
+
#
|
4
|
+
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
|
5
|
+
#
|
6
|
+
|
7
|
+
require 'fb303_types'
|
8
|
+
|
9
|
+
|
10
|
+
module Jp
|
11
|
+
class Job
|
12
|
+
include ::Thrift::Struct, ::Thrift::Struct_Union
|
13
|
+
MESSAGE = 1
|
14
|
+
ID = 2
|
15
|
+
|
16
|
+
FIELDS = {
|
17
|
+
MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message', :binary => true},
|
18
|
+
ID => {:type => ::Thrift::Types::STRING, :name => 'id', :binary => true}
|
19
|
+
}
|
20
|
+
|
21
|
+
def struct_fields; FIELDS; end
|
22
|
+
|
23
|
+
def validate
|
24
|
+
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field message is unset!') unless @message
|
25
|
+
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field id is unset!') unless @id
|
26
|
+
end
|
27
|
+
|
28
|
+
::Thrift::Struct.generate_accessors self
|
29
|
+
end
|
30
|
+
|
31
|
+
class EmptyPool < ::Thrift::Exception
|
32
|
+
include ::Thrift::Struct, ::Thrift::Struct_Union
|
33
|
+
|
34
|
+
FIELDS = {
|
35
|
+
|
36
|
+
}
|
37
|
+
|
38
|
+
def struct_fields; FIELDS; end
|
39
|
+
|
40
|
+
def validate
|
41
|
+
end
|
42
|
+
|
43
|
+
::Thrift::Struct.generate_accessors self
|
44
|
+
end
|
45
|
+
|
46
|
+
class NoSuchPool < ::Thrift::Exception
|
47
|
+
include ::Thrift::Struct, ::Thrift::Struct_Union
|
48
|
+
|
49
|
+
FIELDS = {
|
50
|
+
|
51
|
+
}
|
52
|
+
|
53
|
+
def struct_fields; FIELDS; end
|
54
|
+
|
55
|
+
def validate
|
56
|
+
end
|
57
|
+
|
58
|
+
::Thrift::Struct.generate_accessors self
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'jp/client'
|
4
|
+
|
5
|
+
autoload :JSON, 'json'
|
6
|
+
|
7
|
+
module Jp
|
8
|
+
class AbstractProducer < AbstractClient
|
9
|
+
def initialize queue, options = {}
|
10
|
+
super queue, options
|
11
|
+
end
|
12
|
+
def add message
|
13
|
+
@client.add @queue, translate(message)
|
14
|
+
end
|
15
|
+
private
|
16
|
+
def translate message
|
17
|
+
raise NotImplementedError.new
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
class TextProducer < AbstractProducer
|
22
|
+
private
|
23
|
+
def translate message
|
24
|
+
message
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
class JsonProducer < AbstractProducer
|
29
|
+
private
|
30
|
+
def translate message
|
31
|
+
JSON::dump message
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
class ThriftProducer < AbstractProducer
|
36
|
+
def initialize queue, options = {}
|
37
|
+
super queue, options
|
38
|
+
@serializer = Thrift::Serializer.new
|
39
|
+
end
|
40
|
+
private
|
41
|
+
def translate message
|
42
|
+
@serializer.serialize message
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
autoload :YAML, 'yaml'
|
4
|
+
|
5
|
+
# Find a config file and load it
|
6
|
+
module Jp
|
7
|
+
module Server
|
8
|
+
module ConfigurationLoader
|
9
|
+
def self.server_config
|
10
|
+
name = config_file('jp-config')
|
11
|
+
if name =~ /.ya?ml$/
|
12
|
+
load_yaml_config_file(name)
|
13
|
+
else
|
14
|
+
load_ruby_config_file(name)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
def self.load_yaml_config_file file
|
20
|
+
config = YAML.load(File.read(file))
|
21
|
+
symbolize_hash(config)
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.symbolize_hash hash
|
25
|
+
hash.dup.each do |k,v|
|
26
|
+
if v.is_a? Hash
|
27
|
+
symbolize_hash v
|
28
|
+
end
|
29
|
+
if k.is_a? String
|
30
|
+
hash[k.to_sym] = v
|
31
|
+
end
|
32
|
+
end
|
33
|
+
hash
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.load_ruby_config_file file
|
37
|
+
load file
|
38
|
+
c = JpConfig.new
|
39
|
+
def c.options
|
40
|
+
m = self.public_methods - Object.public_methods - [:options]
|
41
|
+
h = Hash.new
|
42
|
+
m.each do |method|
|
43
|
+
h[method] = self.send method
|
44
|
+
end
|
45
|
+
h
|
46
|
+
end
|
47
|
+
c.options
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.config_file name
|
51
|
+
if ARGV[0] && File.exists?(ARGV[0])
|
52
|
+
return File.expand_path ARGV[0]
|
53
|
+
end
|
54
|
+
|
55
|
+
# Prefixes
|
56
|
+
[
|
57
|
+
'./',
|
58
|
+
'~/',
|
59
|
+
'~/.',
|
60
|
+
'/etc/',
|
61
|
+
].each do |prefix|
|
62
|
+
full = File.expand_path "%s%s.rb" % [prefix, name]
|
63
|
+
if File.exists? full
|
64
|
+
return full
|
65
|
+
end
|
66
|
+
end
|
67
|
+
raise 'No config file found.'
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
require 'jp/thrift'
|
2
|
+
require 'jp/server/mongo_connection'
|
3
|
+
require 'jp/server/pools'
|
4
|
+
|
5
|
+
module Jp
|
6
|
+
module Server
|
7
|
+
# Implementation class.
|
8
|
+
#
|
9
|
+
# Does not include Thrift code, etc.
|
10
|
+
class Handler
|
11
|
+
include MongoConnection
|
12
|
+
include Pools
|
13
|
+
attr_reader :retry_attempts, :retry_delay
|
14
|
+
def initialize options = {}
|
15
|
+
# Option defaults
|
16
|
+
defaults = {
|
17
|
+
:mongo_retry_attempts => 10,
|
18
|
+
:mongo_retry_delay => 1,
|
19
|
+
}
|
20
|
+
options = defaults.merge(options)
|
21
|
+
# Copy with/deal with options
|
22
|
+
@retry_attempts = options[:mongo_retry_attempts]
|
23
|
+
@retry_delay = options[:mongo_retry_delay]
|
24
|
+
|
25
|
+
load_pools(options)
|
26
|
+
connect_to_mongo(options)
|
27
|
+
end
|
28
|
+
|
29
|
+
def add pool, message
|
30
|
+
raise NoSuchPool.new unless pools.member? pool
|
31
|
+
|
32
|
+
doc = {
|
33
|
+
'message' => message,
|
34
|
+
'locked' => false,
|
35
|
+
}
|
36
|
+
|
37
|
+
rescue_connection_failure do
|
38
|
+
database[pool].insert doc
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def acquire pool
|
43
|
+
raise NoSuchPool.new unless pools.member? pool
|
44
|
+
now = Time.new.to_i
|
45
|
+
doc = {}
|
46
|
+
begin
|
47
|
+
rescue_connection_failure do
|
48
|
+
doc = database[pool].find_and_modify(
|
49
|
+
query: {
|
50
|
+
'locked' => false
|
51
|
+
},
|
52
|
+
update: {
|
53
|
+
'$set' => {
|
54
|
+
'locked' => now,
|
55
|
+
'locked_until' => now + pools[pool][:timeout],
|
56
|
+
},
|
57
|
+
}
|
58
|
+
)
|
59
|
+
end
|
60
|
+
rescue Mongo::OperationFailure => e
|
61
|
+
raise EmptyPool
|
62
|
+
end
|
63
|
+
job = Job.new
|
64
|
+
job.message = doc['message']
|
65
|
+
job.id = doc['_id'].to_s
|
66
|
+
job
|
67
|
+
end
|
68
|
+
|
69
|
+
def purge pool, id
|
70
|
+
raise NoSuchPool.new unless pools.member? pool
|
71
|
+
rescue_connection_failure do
|
72
|
+
database[pool].remove _id: BSON::ObjectId(id)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
# Ensure retry upon failure
|
77
|
+
# Based on code from http://www.mongodb.org/display/DOCS/Replica+Pairs+in+Ruby
|
78
|
+
def rescue_connection_failure
|
79
|
+
success = false
|
80
|
+
retries = 0
|
81
|
+
while !success
|
82
|
+
begin
|
83
|
+
yield
|
84
|
+
success = true
|
85
|
+
rescue Mongo::ConnectionFailure => ex
|
86
|
+
retries += 1
|
87
|
+
raise ex if retries >= @retry_attempts
|
88
|
+
sleep(@retry_delay)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
require 'jp/server/server'
|
2
|
+
|
3
|
+
module Jp
|
4
|
+
module Server
|
5
|
+
class InstrumentedServer < Jp::Server::Server
|
6
|
+
def initialize options = {}
|
7
|
+
options[:jp_server] ||= Jp::Server::Server.new options.merge(thrift_processor: JobPoolInstrumented::Processor.new(self))
|
8
|
+
@server = options[:jp_server]
|
9
|
+
@pools = options[:pools].keys
|
10
|
+
@add_count = Hash.new 0
|
11
|
+
@acquire_count = Hash.new 0
|
12
|
+
@purge_count = Hash.new 0
|
13
|
+
@empty_count = Hash.new 0
|
14
|
+
end
|
15
|
+
|
16
|
+
# Readers
|
17
|
+
|
18
|
+
def start_time
|
19
|
+
@server.aliveSince
|
20
|
+
end
|
21
|
+
|
22
|
+
def pools
|
23
|
+
@pools
|
24
|
+
end
|
25
|
+
|
26
|
+
def add_count pool
|
27
|
+
raise Jp::NoSuchPool.new unless @pools.include? pool
|
28
|
+
@add_count[pool]
|
29
|
+
end
|
30
|
+
|
31
|
+
def acquire_count pool
|
32
|
+
raise Jp::NoSuchPool.new unless @pools.include? pool
|
33
|
+
@acquire_count[pool]
|
34
|
+
end
|
35
|
+
|
36
|
+
def empty_count pool
|
37
|
+
raise Jp::NoSuchPool.new unless @pools.include? pool
|
38
|
+
@empty_count[pool]
|
39
|
+
end
|
40
|
+
|
41
|
+
def purge_count pool
|
42
|
+
raise Jp::NoSuchPool.new unless @pools.include? pool
|
43
|
+
@purge_count[pool]
|
44
|
+
end
|
45
|
+
|
46
|
+
# Data collectors
|
47
|
+
|
48
|
+
def serve
|
49
|
+
@server.serve
|
50
|
+
end
|
51
|
+
|
52
|
+
def add pool, message
|
53
|
+
result = @server.add pool, message
|
54
|
+
@add_count[pool] += 1
|
55
|
+
result
|
56
|
+
end
|
57
|
+
|
58
|
+
def acquire pool
|
59
|
+
begin
|
60
|
+
result = @server.acquire pool
|
61
|
+
@acquire_count[pool] += 1
|
62
|
+
rescue EmptyPool => e
|
63
|
+
@empty_count[pool] += 1
|
64
|
+
raise e
|
65
|
+
end
|
66
|
+
result
|
67
|
+
end
|
68
|
+
|
69
|
+
def purge pool, id
|
70
|
+
result = @server.purge pool, id
|
71
|
+
@purge_count[pool] += 1
|
72
|
+
result
|
73
|
+
end
|
74
|
+
|
75
|
+
def getCounters
|
76
|
+
counters = Hash.new
|
77
|
+
pools.each do |pool|
|
78
|
+
counters["#{pool}.added"] = add_count(pool)
|
79
|
+
counters["#{pool}.acquired"] = acquire_count(pool)
|
80
|
+
counters["#{pool}.empty"] = empty_count(pool)
|
81
|
+
counters["#{pool}.purged"] = purge_count(pool)
|
82
|
+
end
|
83
|
+
counters
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Jp
|
2
|
+
module Server
|
3
|
+
module MongoConnection
|
4
|
+
attr_reader :database
|
5
|
+
def connect_to_mongo options
|
6
|
+
defaults = {
|
7
|
+
:mongo_uri => 'mongodb://localhost',
|
8
|
+
:mongo_pool_size => 10,
|
9
|
+
:mongo_pool_timeout => 60,
|
10
|
+
}
|
11
|
+
options = defaults.merge(options)
|
12
|
+
unless options[:mongo_db]
|
13
|
+
raise ArgumentError.new "mongo_db option must be specified"
|
14
|
+
end
|
15
|
+
# Connect to mongodb
|
16
|
+
if options.member? :injected_mongo_database then
|
17
|
+
@database = options[:injected_mongo_database]
|
18
|
+
else
|
19
|
+
connection = Mongo::Connection.from_uri(
|
20
|
+
options[:mongo_uri],
|
21
|
+
:pool_size => options[:mongo_pool_size],
|
22
|
+
:timeout => options[:mongo_pool_timeout],
|
23
|
+
)
|
24
|
+
@database = connection.db(options[:mongo_db])
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Jp
|
2
|
+
module Server
|
3
|
+
module Pools
|
4
|
+
attr_reader :pools
|
5
|
+
protected
|
6
|
+
def load_pools options
|
7
|
+
defaults = {
|
8
|
+
:default_timeout => 3600, # 1 hour
|
9
|
+
}
|
10
|
+
options = defaults.merge(options)
|
11
|
+
|
12
|
+
unless options[:pools]
|
13
|
+
raise ArgumentError.new "pools option must be specified"
|
14
|
+
end
|
15
|
+
if options[:pools].empty?
|
16
|
+
raise ArgumentError.new "pools option must not be empty"
|
17
|
+
end
|
18
|
+
|
19
|
+
@pools = Hash.new
|
20
|
+
options[:pools].each do |name, data|
|
21
|
+
data[:timeout] ||= options[:default_timeout]
|
22
|
+
data[:cleanup_interval] ||= data[:timeout]
|
23
|
+
@pools[name] = data
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'jp/thrift'
|
2
|
+
require 'jp/server/handler'
|
3
|
+
require 'jp/server/unlocker'
|
4
|
+
|
5
|
+
require 'mongo'
|
6
|
+
include Jp
|
7
|
+
|
8
|
+
module Jp
|
9
|
+
module Server
|
10
|
+
class Server < Jp::Server::Handler
|
11
|
+
def initialize options = {}
|
12
|
+
super options
|
13
|
+
options[:port_number] ||= 9090
|
14
|
+
|
15
|
+
# Setup Thrift server (allowing dependency injection)
|
16
|
+
if options.member? :injected_thrift_server then
|
17
|
+
@server = options[:injected_thrift_server]
|
18
|
+
else
|
19
|
+
processor = options[:thrift_processor] # For testing, and allow instrumented server to override
|
20
|
+
processor ||= JobPool::Processor.new self
|
21
|
+
socket = Thrift::ServerSocket.new options[:port_number]
|
22
|
+
transportFactory = Thrift::BufferedTransportFactory.new
|
23
|
+
|
24
|
+
@server = Thrift::ThreadedServer.new processor, socket, transportFactory
|
25
|
+
end
|
26
|
+
|
27
|
+
@unlocker = nil
|
28
|
+
unless options[:skip_embedded_unlocker]
|
29
|
+
if options.member? :injected_unlocker
|
30
|
+
@unlocker = options[:injected_unlocker]
|
31
|
+
else
|
32
|
+
@unlocker = Jp::Server::Unlocker.new options
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
@start_time = Time.new.to_i
|
37
|
+
end
|
38
|
+
|
39
|
+
def serve
|
40
|
+
# Look for expired entries
|
41
|
+
@unlocker ||= nil
|
42
|
+
unlocker_thread = nil
|
43
|
+
if @unlocker
|
44
|
+
unlocker_thread = Thread.new do
|
45
|
+
@unlocker.run
|
46
|
+
end
|
47
|
+
end
|
48
|
+
@server.serve
|
49
|
+
unlocker_thread.join if unlocker_thread
|
50
|
+
end
|
51
|
+
|
52
|
+
|
53
|
+
# fb303:
|
54
|
+
def getName; 'jp'; end
|
55
|
+
def getVersion; '0.0.1'; end
|
56
|
+
def getStatus; Fb_status::ALIVE; end
|
57
|
+
def getStatusDetails; 'nothing to see here; move along'; end
|
58
|
+
def aliveSince; @start_time; end
|
59
|
+
def shutdown
|
60
|
+
STDERR.write "Shutdown requested via fb303\n"
|
61
|
+
exit
|
62
|
+
end
|
63
|
+
# fb303 stubs:
|
64
|
+
def setOption(key, value); end
|
65
|
+
def getOption(key); end
|
66
|
+
def getOptions; Hash.new; end
|
67
|
+
def getCpuProfile(seconds); String.new; end
|
68
|
+
def reinitialize; end
|
69
|
+
# fb303 stubs properly implemented in JpInstrumentedServer:
|
70
|
+
def getCounters; Hash.new; end
|
71
|
+
def getCounter(name); 0; end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|