starling-starling 0.9.7.9

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,151 @@
1
+ module StarlingServer
2
+
3
+ ##
4
+ # PersistentQueue is a subclass of Ruby's thread-safe Queue class. It adds a
5
+ # transactional log to the in-memory Queue, which enables quickly rebuilding
6
+ # the Queue in the event of a sever outage.
7
+
8
+ class PersistentQueue < Queue
9
+
10
+ ##
11
+ # When a log reaches the SOFT_LOG_MAX_SIZE, the Queue will wait until
12
+ # it is empty, and will then rotate the log file.
13
+
14
+ SOFT_LOG_MAX_SIZE = 16 * (1024**2) # 16 MB
15
+
16
+ TRX_CMD_PUSH = "\000".freeze
17
+ TRX_CMD_POP = "\001".freeze
18
+
19
+ TRX_PUSH = "\000%s%s".freeze
20
+ TRX_POP = "\001".freeze
21
+
22
+ attr_reader :initial_bytes
23
+ attr_reader :total_items
24
+ attr_reader :logsize
25
+ attr_reader :current_age
26
+
27
+ ##
28
+ # Create a new PersistentQueue at +persistence_path+/+queue_name+.
29
+ # If a queue log exists at that path, the Queue will be loaded from
30
+ # disk before being available for use.
31
+
32
+ def initialize(persistence_path, queue_name, debug = false)
33
+ @persistence_path = persistence_path
34
+ @queue_name = queue_name
35
+ @total_items = 0
36
+ super()
37
+ @initial_bytes = replay_transaction_log(debug)
38
+ @current_age = 0
39
+ end
40
+
41
+ ##
42
+ # Pushes +value+ to the queue. By default, +push+ will write to the
43
+ # transactional log. Set +log_trx=false+ to override this behaviour.
44
+
45
+ def push(value, log_trx = true)
46
+ if log_trx
47
+ raise NoTransactionLog unless @trx
48
+ size = [value.size].pack("I")
49
+ transaction sprintf(TRX_PUSH, size, value)
50
+ end
51
+
52
+ @total_items += 1
53
+ super([now_usec, value])
54
+ end
55
+
56
+ ##
57
+ # Retrieves data from the queue.
58
+
59
+ def pop(log_trx = true)
60
+ raise NoTransactionLog if log_trx && !@trx
61
+
62
+ begin
63
+ rv = super(!log_trx)
64
+ rescue ThreadError
65
+ puts "WARNING: The queue was empty when trying to pop(). Technically this shouldn't ever happen. Probably a bug in the transactional underpinnings. Or maybe shutdown didn't happen cleanly at some point. Ignoring."
66
+ rv = [now_usec, '']
67
+ end
68
+ transaction "\001" if log_trx
69
+ @current_age = (now_usec - rv[0]) / 1000
70
+ rv[1]
71
+ end
72
+
73
+ ##
74
+ # Safely closes the transactional queue.
75
+
76
+ def close
77
+ # Ok, yeah, this is lame, and is *technically* a race condition. HOWEVER,
78
+ # the QueueCollection *should* have stopped processing requests, and I don't
79
+ # want to add yet another Mutex around all the push and pop methods. So we
80
+ # do the next simplest thing, and minimize the time we'll stick around before
81
+ # @trx is nil.
82
+ @not_trx = @trx
83
+ @trx = nil
84
+ @not_trx.close
85
+ end
86
+
87
+ private
88
+
89
+ def log_path #:nodoc:
90
+ File.join(@persistence_path, @queue_name)
91
+ end
92
+
93
+ def reopen_log #:nodoc:
94
+ @trx = File.new(log_path, File::CREAT|File::RDWR)
95
+ @logsize = File.size(log_path)
96
+ end
97
+
98
+ def rotate_log #:nodoc:
99
+ @trx.close
100
+ backup_logfile = "#{log_path}.#{Time.now.to_i}"
101
+ File.rename(log_path, backup_logfile)
102
+ reopen_log
103
+ File.unlink(backup_logfile)
104
+ end
105
+
106
+ def replay_transaction_log(debug) #:nodoc:
107
+ reopen_log
108
+ bytes_read = 0
109
+
110
+ print "Reading back transaction log for #{@queue_name} " if debug
111
+
112
+ while !@trx.eof?
113
+ cmd = @trx.read(1)
114
+ case cmd
115
+ when TRX_CMD_PUSH
116
+ print ">" if debug
117
+ raw_size = @trx.read(4)
118
+ next unless raw_size
119
+ size = raw_size.unpack("I").first
120
+ data = @trx.read(size)
121
+ next unless data
122
+ push(data, false)
123
+ bytes_read += data.size
124
+ when TRX_CMD_POP
125
+ print "<" if debug
126
+ bytes_read -= pop(false).size
127
+ else
128
+ puts "Error reading transaction log: " +
129
+ "I don't understand '#{cmd}' (skipping)." if debug
130
+ end
131
+ end
132
+
133
+ print " done.\n" if debug
134
+
135
+ return bytes_read
136
+ end
137
+
138
+ def transaction(data) #:nodoc:
139
+ raise "no transaction log handle. that totally sucks." unless @trx
140
+
141
+ @trx.write_nonblock data
142
+ @logsize += data.size
143
+ rotate_log if @logsize > SOFT_LOG_MAX_SIZE && self.length == 0
144
+ end
145
+
146
+ def now_usec
147
+ now = Time.now
148
+ now.to_i * 1000000 + now.usec
149
+ end
150
+ end
151
+ end
@@ -0,0 +1,141 @@
1
+ require 'thread'
2
+ require 'starling/persistent_queue'
3
+
4
+ module StarlingServer
5
+ class InaccessibleQueuePath < Exception #:nodoc:
6
+ end
7
+
8
+ ##
9
+ # QueueCollection is a proxy to a collection of PersistentQueue instances.
10
+
11
+ class QueueCollection
12
+
13
+ ##
14
+ # Create a new QueueCollection at +path+
15
+
16
+ def initialize(path)
17
+ unless File.directory?(path) && File.writable?(path)
18
+ raise InaccessibleQueuePath.new("#{path} must exist and be writable by #{Etc.getpwuid(Process.uid).name}.")
19
+ end
20
+
21
+ @shutdown_mutex = Mutex.new
22
+
23
+ @path = path
24
+ @logger = StarlingServer::Base.logger
25
+
26
+ @queues = {}
27
+ @queue_init_mutexes = {}
28
+
29
+ @stats = Hash.new(0)
30
+ end
31
+
32
+ ##
33
+ # Puts +data+ onto the queue named +key+
34
+
35
+ def put(key, data)
36
+ queue = queues(key)
37
+ return nil unless queue
38
+
39
+ @stats[:current_bytes] += data.size
40
+ @stats[:total_items] += 1
41
+
42
+ queue.push(data)
43
+
44
+ return true
45
+ end
46
+
47
+ ##
48
+ # Retrieves data from the queue named +key+
49
+
50
+ def take(key)
51
+ queue = queues(key)
52
+ if queue.nil? || queue.length == 0
53
+ @stats[:get_misses] += 1
54
+ return nil
55
+ else
56
+ @stats[:get_hits] += 1
57
+ end
58
+ result = queue.pop
59
+ @stats[:current_bytes] -= result.size
60
+ result
61
+ end
62
+
63
+ ##
64
+ # Returns all active queues.
65
+
66
+ def queues(key=nil)
67
+ return nil if @shutdown_mutex.locked?
68
+
69
+ return @queues if key.nil?
70
+
71
+ # First try to return the queue named 'key' if it's available.
72
+ return @queues[key] if @queues[key]
73
+
74
+ # If the queue wasn't available, create or get the mutex that will
75
+ # wrap creation of the Queue.
76
+ @queue_init_mutexes[key] ||= Mutex.new
77
+
78
+ # Otherwise, check to see if another process is already loading
79
+ # the queue named 'key'.
80
+ if @queue_init_mutexes[key].locked?
81
+ # return an empty/false result if we're waiting for the queue
82
+ # to be loaded and we're not the first process to request the queue
83
+ return nil
84
+ else
85
+ begin
86
+ @queue_init_mutexes[key].lock
87
+ # we've locked the mutex, but only go ahead if the queue hasn't
88
+ # been loaded. There's a race condition otherwise, and we could
89
+ # end up loading the queue multiple times.
90
+ if @queues[key].nil?
91
+ @queues[key] = PersistentQueue.new(@path, key)
92
+ @stats[:current_bytes] += @queues[key].initial_bytes
93
+ end
94
+ rescue Object => exc
95
+ puts "ZOMG There was an exception reading back the queue. That totally sucks."
96
+ puts "The exception was: #{exc}. Backtrace: #{exc.backtrace.join("\n")}"
97
+ ensure
98
+ @queue_init_mutexes[key].unlock
99
+ end
100
+ end
101
+
102
+ return @queues[key]
103
+ end
104
+
105
+ ##
106
+ # Returns statistic +stat_name+ for the QueueCollection.
107
+ #
108
+ # Valid statistics are:
109
+ #
110
+ # [:get_misses] Total number of get requests with empty responses
111
+ # [:get_hits] Total number of get requests that returned data
112
+ # [:current_bytes] Current size in bytes of items in the queues
113
+ # [:current_size] Current number of items across all queues
114
+ # [:total_items] Total number of items stored in queues.
115
+
116
+ def stats(stat_name)
117
+ case stat_name
118
+ when nil; @stats
119
+ when :current_size; current_size
120
+ else; @stats[stat_name]
121
+ end
122
+ end
123
+
124
+ ##
125
+ # Safely close all queues.
126
+
127
+ def close
128
+ @shutdown_mutex.lock
129
+ @queues.each_pair do |name,queue|
130
+ queue.close
131
+ @queues.delete(name)
132
+ end
133
+ end
134
+
135
+ private
136
+
137
+ def current_size #:nodoc:
138
+ @queues.inject(0) { |m, (k,v)| m + v.length }
139
+ end
140
+ end
141
+ end
@@ -0,0 +1,113 @@
1
+ require 'socket'
2
+ require 'logger'
3
+ require 'rubygems'
4
+ require 'eventmachine'
5
+ require 'analyzer_tools/syslog_logger'
6
+
7
+ here = File.dirname(__FILE__)
8
+
9
+ require File.join(here, 'queue_collection')
10
+ require File.join(here, 'handler')
11
+
12
+ module StarlingServer
13
+
14
+ VERSION = "0.9.7.9"
15
+
16
+ class Base
17
+ attr_reader :logger
18
+
19
+ DEFAULT_HOST = '127.0.0.1'
20
+ DEFAULT_PORT = 22122
21
+ DEFAULT_PATH = "/tmp/starling/"
22
+ DEFAULT_TIMEOUT = 60
23
+
24
+ ##
25
+ # Initialize a new Starling server and immediately start processing
26
+ # requests.
27
+ #
28
+ # +opts+ is an optional hash, whose valid options are:
29
+ #
30
+ # [:host] Host on which to listen (default is 127.0.0.1).
31
+ # [:port] Port on which to listen (default is 22122).
32
+ # [:path] Path to Starling queue logs. Default is /tmp/starling/
33
+ # [:timeout] Time in seconds to wait before closing connections.
34
+ # [:logger] A Logger object, an IO handle, or a path to the log.
35
+ # [:loglevel] Logger verbosity. Default is Logger::ERROR.
36
+ #
37
+ # Other options are ignored.
38
+
39
+ def self.start(opts = {})
40
+ server = self.new(opts)
41
+ server.run
42
+ end
43
+
44
+ ##
45
+ # Initialize a new Starling server, but do not accept connections or
46
+ # process requests.
47
+ #
48
+ # +opts+ is as for +start+
49
+
50
+ def initialize(opts = {})
51
+ @opts = {
52
+ :host => DEFAULT_HOST,
53
+ :port => DEFAULT_PORT,
54
+ :path => DEFAULT_PATH,
55
+ :timeout => DEFAULT_TIMEOUT,
56
+ :server => self
57
+ }.merge(opts)
58
+
59
+ @stats = Hash.new(0)
60
+
61
+ FileUtils.mkdir_p(@opts[:path])
62
+
63
+ end
64
+
65
+ ##
66
+ # Start listening and processing requests.
67
+
68
+ def run
69
+ @stats[:start_time] = Time.now
70
+
71
+ @@logger = case @opts[:logger]
72
+ when IO, String; Logger.new(@opts[:logger])
73
+ when Logger; @opts[:logger]
74
+ else; Logger.new(STDERR)
75
+ end
76
+ @@logger = SyslogLogger.new(@opts[:syslog_channel]) if @opts[:syslog_channel]
77
+
78
+ @opts[:queue] = QueueCollection.new(@opts[:path])
79
+ @@logger.level = @opts[:log_level] || Logger::ERROR
80
+
81
+ @@logger.info "Starling STARTUP on #{@opts[:host]}:#{@opts[:port]}"
82
+
83
+ EventMachine.epoll
84
+ EventMachine.set_descriptor_table_size(4096)
85
+ EventMachine.run do
86
+ EventMachine.start_server(@opts[:host], @opts[:port], Handler, @opts)
87
+ end
88
+
89
+ # code here will get executed on shutdown:
90
+ @opts[:queue].close
91
+ end
92
+
93
+ def self.logger
94
+ @@logger
95
+ end
96
+
97
+
98
+ ##
99
+ # Stop accepting new connections and shutdown gracefully.
100
+
101
+ def stop
102
+ EventMachine.stop_event_loop
103
+ end
104
+
105
+ def stats(stat = nil) #:nodoc:
106
+ case stat
107
+ when nil; @stats
108
+ when :connections; 1
109
+ else; @stats[stat]
110
+ end
111
+ end
112
+ end
113
+ end
@@ -0,0 +1,292 @@
1
+ require File.join(File.dirname(__FILE__), 'server')
2
+ require 'optparse'
3
+ require 'yaml'
4
+
5
+ module StarlingServer
6
+ class Runner
7
+
8
+ attr_accessor :options
9
+ private :options, :options=
10
+
11
+ def self.run
12
+ new
13
+ end
14
+
15
+ def self.shutdown
16
+ @@instance.shutdown
17
+ end
18
+
19
+ def initialize
20
+ @@instance = self
21
+ parse_options
22
+
23
+ @process = ProcessHelper.new(options[:logger], options[:pid_file], options[:user], options[:group])
24
+
25
+ pid = @process.running?
26
+ if pid
27
+ STDERR.puts "There is already a Starling process running (pid #{pid}), exiting."
28
+ exit(1)
29
+ elsif pid.nil?
30
+ STDERR.puts "Cleaning up stale pidfile at #{options[:pid_file]}."
31
+ end
32
+
33
+ start
34
+ end
35
+
36
+ def load_config_file(filename)
37
+ YAML.load(File.open(filename))['starling'].each do |key, value|
38
+ # alias some keys
39
+ case key
40
+ when "queue_path" then key = "path"
41
+ when "log_file" then key = "logger"
42
+ end
43
+ options[key.to_sym] = value
44
+
45
+ if options[:log_level].instance_of?(String)
46
+ options[:log_level] = Logger.const_get(options[:log_level])
47
+ end
48
+ end
49
+ end
50
+
51
+ def parse_options
52
+ self.options = { :host => '127.0.0.1',
53
+ :port => 22122,
54
+ :path => File.join(%w( / tmp starling spool )),
55
+ :log_level => Logger::INFO,
56
+ :daemonize => false,
57
+ :timeout => 0,
58
+ :pid_file => File.join(%w( / tmp starling starling.pid )) }
59
+
60
+ OptionParser.new do |opts|
61
+ opts.summary_width = 25
62
+
63
+ opts.banner = "Starling (#{StarlingServer::VERSION})\n\n",
64
+ "usage: starling [options...]\n",
65
+ " starling --help\n",
66
+ " starling --version\n"
67
+
68
+ opts.separator ""
69
+ opts.separator "Configuration:"
70
+
71
+ opts.on("-f", "--config FILENAME",
72
+ "Config file (yaml) to load") do |filename|
73
+ load_config_file(filename)
74
+ end
75
+
76
+ opts.on("-q", "--queue_path PATH",
77
+ :REQUIRED,
78
+ "Path to store Starling queue logs", "(default: #{options[:path]})") do |queue_path|
79
+ options[:path] = queue_path
80
+ end
81
+
82
+ opts.separator ""; opts.separator "Network:"
83
+
84
+ opts.on("-hHOST", "--host HOST", "Interface on which to listen (default: #{options[:host]})") do |host|
85
+ options[:host] = host
86
+ end
87
+
88
+ opts.on("-pHOST", "--port PORT", Integer, "TCP port on which to listen (default: #{options[:port]})") do |port|
89
+ options[:port] = port
90
+ end
91
+
92
+ opts.separator ""; opts.separator "Process:"
93
+
94
+ opts.on("-d", "Run as a daemon.") do
95
+ options[:daemonize] = true
96
+ end
97
+
98
+ opts.on("-PFILE", "--pid FILENAME", "save PID in FILENAME when using -d option.", "(default: #{options[:pid_file]})") do |pid_file|
99
+ options[:pid_file] = pid_file
100
+ end
101
+
102
+ opts.on("-u", "--user USER", "User to run as") do |user|
103
+ options[:user] = user.to_i == 0 ? Etc.getpwnam(user).uid : user.to_i
104
+ end
105
+
106
+ opts.on("-gGROUP", "--group GROUP", "Group to run as") do |group|
107
+ options[:group] = group.to_i == 0 ? Etc.getgrnam(group).gid : group.to_i
108
+ end
109
+
110
+ opts.separator ""; opts.separator "Logging:"
111
+
112
+ opts.on("-L", "--log [FILE]", "Path to print debugging information.") do |log_path|
113
+ options[:logger] = log_path
114
+ end
115
+
116
+ opts.on("-l", "--syslog CHANNEL", "Write logs to the syslog instead of a log file.") do |channel|
117
+ options[:syslog_channel] = channel
118
+ end
119
+
120
+ opts.on("-v", "Increase logging verbosity (may be used multiple times).") do
121
+ options[:log_level] -= 1
122
+ end
123
+
124
+ opts.on("-t", "--timeout [SECONDS]", Integer,
125
+ "Time in seconds before disconnecting inactive clients (0 to disable).",
126
+ "(default: #{options[:timeout]})") do |timeout|
127
+ options[:timeout] = timeout
128
+ end
129
+
130
+ opts.separator ""; opts.separator "Miscellaneous:"
131
+
132
+ opts.on_tail("-?", "--help", "Display this usage information.") do
133
+ puts "#{opts}\n"
134
+ exit
135
+ end
136
+
137
+ opts.on_tail("-V", "--version", "Print version number and exit.") do
138
+ puts "Starling #{StarlingServer::VERSION}\n\n"
139
+ exit
140
+ end
141
+ end.parse!
142
+ end
143
+
144
+ def start
145
+ drop_privileges
146
+
147
+ @process.daemonize if options[:daemonize]
148
+
149
+ setup_signal_traps
150
+ @process.write_pid_file
151
+
152
+ STDOUT.puts "Starting at #{options[:host]}:#{options[:port]}."
153
+ @server = StarlingServer::Base.new(options)
154
+ @server.run
155
+
156
+ @process.remove_pid_file
157
+ end
158
+
159
+ def drop_privileges
160
+ Process.egid = options[:group] if options[:group]
161
+ Process.euid = options[:user] if options[:user]
162
+ end
163
+
164
+ def shutdown
165
+ begin
166
+ STDOUT.puts "Shutting down."
167
+ StarlingServer::Base.logger.info "Shutting down."
168
+ @server.stop
169
+ rescue Object => e
170
+ STDERR.puts "There was an error shutting down: #{e}"
171
+ exit(70)
172
+ end
173
+ end
174
+
175
+ def setup_signal_traps
176
+ Signal.trap("INT") { shutdown }
177
+ Signal.trap("TERM") { shutdown }
178
+ end
179
+ end
180
+
181
+ class ProcessHelper
182
+
183
+ def initialize(log_file = nil, pid_file = nil, user = nil, group = nil)
184
+ @log_file = log_file
185
+ @pid_file = pid_file
186
+ @user = user
187
+ @group = group
188
+ end
189
+
190
+ def safefork
191
+ begin
192
+ if pid = fork
193
+ return pid
194
+ end
195
+ rescue Errno::EWOULDBLOCK
196
+ sleep 5
197
+ retry
198
+ end
199
+ end
200
+
201
+ def daemonize
202
+ sess_id = detach_from_terminal
203
+ exit if pid = safefork
204
+
205
+ Dir.chdir("/")
206
+ File.umask 0000
207
+
208
+ close_io_handles
209
+ redirect_io
210
+
211
+ return sess_id
212
+ end
213
+
214
+ def detach_from_terminal
215
+ srand
216
+ safefork and exit
217
+
218
+ unless sess_id = Process.setsid
219
+ raise "Couldn't detache from controlling terminal."
220
+ end
221
+
222
+ trap 'SIGHUP', 'IGNORE'
223
+
224
+ sess_id
225
+ end
226
+
227
+ def close_io_handles
228
+ ObjectSpace.each_object(IO) do |io|
229
+ unless [STDIN, STDOUT, STDERR].include?(io)
230
+ begin
231
+ io.close unless io.closed?
232
+ rescue Exception
233
+ end
234
+ end
235
+ end
236
+ end
237
+
238
+ def redirect_io
239
+ begin; STDIN.reopen('/dev/null'); rescue Exception; end
240
+
241
+ if @log_file
242
+ begin
243
+ STDOUT.reopen(@log_file, "a")
244
+ STDOUT.sync = true
245
+ rescue Exception
246
+ begin; STDOUT.reopen('/dev/null'); rescue Exception; end
247
+ end
248
+ else
249
+ begin; STDOUT.reopen('/dev/null'); rescue Exception; end
250
+ end
251
+
252
+ begin; STDERR.reopen(STDOUT); rescue Exception; end
253
+ STDERR.sync = true
254
+ end
255
+
256
+ def rescue_exception
257
+ begin
258
+ yield
259
+ rescue Exception
260
+ end
261
+ end
262
+
263
+ def write_pid_file
264
+ return unless @pid_file
265
+ FileUtils.mkdir_p(File.dirname(@pid_file))
266
+ File.open(@pid_file, "w") { |f| f.write(Process.pid) }
267
+ File.chmod(0644, @pid_file)
268
+ end
269
+
270
+ def remove_pid_file
271
+ return unless @pid_file
272
+ File.unlink(@pid_file) if File.exists?(@pid_file)
273
+ end
274
+
275
+ def running?
276
+ return false unless @pid_file
277
+
278
+ pid = File.read(@pid_file).chomp.to_i rescue nil
279
+ pid = nil if pid == 0
280
+ return false unless pid
281
+
282
+ begin
283
+ Process.kill(0, pid)
284
+ return pid
285
+ rescue Errno::ESRCH
286
+ return nil
287
+ rescue Errno::EPERM
288
+ return pid
289
+ end
290
+ end
291
+ end
292
+ end