myobie-starling 0.9.10

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,147 @@
1
+ require 'thread'
2
+ require 'starling/persistent_queue'
3
+
4
+ module StarlingServer
5
+ class InaccessibleQueuePath < Exception #:nodoc:
6
+ end
7
+
8
+ ##
9
+ # QueueCollection is a proxy to a collection of PersistentQueue instances.
10
+
11
+ class QueueCollection
12
+
13
+ ##
14
+ # Create a new QueueCollection at +path+
15
+
16
+ def initialize(path)
17
+ unless File.directory?(path) && File.writable?(path)
18
+ raise InaccessibleQueuePath.new("'#{path}' must exist and be read-writable by #{Etc.getpwuid(Process.uid).name}.")
19
+ end
20
+
21
+ @shutdown_mutex = Mutex.new
22
+
23
+ @path = path
24
+ @logger = StarlingServer::Base.logger
25
+
26
+ @queues = {}
27
+ @queue_init_mutexes = {}
28
+
29
+ @stats = Hash.new(0)
30
+ end
31
+
32
+ ##
33
+ # Puts +data+ onto the queue named +key+
34
+
35
+ def put(key, data)
36
+ queue = queues(key)
37
+ return nil unless queue
38
+
39
+ @stats[:current_bytes] += data.size
40
+ @stats[:total_items] += 1
41
+
42
+ queue.push(data)
43
+
44
+ return true
45
+ end
46
+
47
+ ##
48
+ # Retrieves data from the queue named +key+
49
+
50
+ def take(key)
51
+ queue = queues(key)
52
+ if queue.nil? || queue.length == 0
53
+ @stats[:get_misses] += 1
54
+ return nil
55
+ else
56
+ @stats[:get_hits] += 1
57
+ end
58
+ result = queue.pop
59
+ @stats[:current_bytes] -= result.size
60
+ result
61
+ end
62
+
63
+ def delete(key)
64
+ queue = @queues.delete(key)
65
+ return if queue.nil?
66
+ queue.purge
67
+ end
68
+
69
+ ##
70
+ # Returns all active queues.
71
+
72
+ def queues(key=nil)
73
+ return nil if @shutdown_mutex.locked?
74
+
75
+ return @queues if key.nil?
76
+
77
+ # First try to return the queue named 'key' if it's available.
78
+ return @queues[key] if @queues[key]
79
+
80
+ # If the queue wasn't available, create or get the mutex that will
81
+ # wrap creation of the Queue.
82
+ @queue_init_mutexes[key] ||= Mutex.new
83
+
84
+ # Otherwise, check to see if another process is already loading
85
+ # the queue named 'key'.
86
+ if @queue_init_mutexes[key].locked?
87
+ # return an empty/false result if we're waiting for the queue
88
+ # to be loaded and we're not the first process to request the queue
89
+ return nil
90
+ else
91
+ begin
92
+ @queue_init_mutexes[key].lock
93
+ # we've locked the mutex, but only go ahead if the queue hasn't
94
+ # been loaded. There's a race condition otherwise, and we could
95
+ # end up loading the queue multiple times.
96
+ if @queues[key].nil?
97
+ @queues[key] = PersistentQueue.new(@path, key)
98
+ @stats[:current_bytes] += @queues[key].initial_bytes
99
+ end
100
+ rescue Object => exc
101
+ puts "ZOMG There was an exception reading back the queue. That totally sucks."
102
+ puts "The exception was: #{exc}. Backtrace: #{exc.backtrace.join("\n")}"
103
+ ensure
104
+ @queue_init_mutexes[key].unlock
105
+ end
106
+ end
107
+
108
+ return @queues[key]
109
+ end
110
+
111
+ ##
112
+ # Returns statistic +stat_name+ for the QueueCollection.
113
+ #
114
+ # Valid statistics are:
115
+ #
116
+ # [:get_misses] Total number of get requests with empty responses
117
+ # [:get_hits] Total number of get requests that returned data
118
+ # [:current_bytes] Current size in bytes of items in the queues
119
+ # [:current_size] Current number of items across all queues
120
+ # [:total_items] Total number of items stored in queues.
121
+
122
+ def stats(stat_name)
123
+ case stat_name
124
+ when nil; @stats
125
+ when :current_size; current_size
126
+ else; @stats[stat_name]
127
+ end
128
+ end
129
+
130
+ ##
131
+ # Safely close all queues.
132
+
133
+ def close
134
+ @shutdown_mutex.lock
135
+ @queues.each_pair do |name,queue|
136
+ queue.close
137
+ @queues.delete(name)
138
+ end
139
+ end
140
+
141
+ private
142
+
143
+ def current_size #:nodoc:
144
+ @queues.inject(0) { |m, (k,v)| m + v.length }
145
+ end
146
+ end
147
+ end
@@ -0,0 +1,125 @@
1
+ require 'socket'
2
+ require 'logger'
3
+ require 'rubygems'
4
+ require 'eventmachine'
5
+
6
+ here = File.dirname(__FILE__)
7
+
8
+ require File.join(here, 'queue_collection')
9
+ require File.join(here, 'handler')
10
+
11
+ module StarlingServer
12
+
13
+ VERSION = "0.9.9"
14
+
15
+ class Base
16
+ attr_reader :logger
17
+
18
+ DEFAULT_HOST = '127.0.0.1'
19
+ DEFAULT_PORT = 22122
20
+ DEFAULT_PATH = "/tmp/starling/"
21
+ DEFAULT_TIMEOUT = 60
22
+
23
+ ##
24
+ # Initialize a new Starling server and immediately start processing
25
+ # requests.
26
+ #
27
+ # +opts+ is an optional hash, whose valid options are:
28
+ #
29
+ # [:host] Host on which to listen (default is 127.0.0.1).
30
+ # [:port] Port on which to listen (default is 22122).
31
+ # [:path] Path to Starling queue logs. Default is /tmp/starling/
32
+ # [:timeout] Time in seconds to wait before closing connections.
33
+ # [:logger] A Logger object, an IO handle, or a path to the log.
34
+ # [:loglevel] Logger verbosity. Default is Logger::ERROR.
35
+ #
36
+ # Other options are ignored.
37
+
38
+ def self.start(opts = {})
39
+ server = self.new(opts)
40
+ server.run
41
+ end
42
+
43
+ ##
44
+ # Initialize a new Starling server, but do not accept connections or
45
+ # process requests.
46
+ #
47
+ # +opts+ is as for +start+
48
+
49
+ def initialize(opts = {})
50
+ @opts = {
51
+ :host => DEFAULT_HOST,
52
+ :port => DEFAULT_PORT,
53
+ :path => DEFAULT_PATH,
54
+ :timeout => DEFAULT_TIMEOUT,
55
+ :server => self
56
+ }.merge(opts)
57
+
58
+ @stats = Hash.new(0)
59
+
60
+ FileUtils.mkdir_p(@opts[:path])
61
+
62
+ end
63
+
64
+ ##
65
+ # Start listening and processing requests.
66
+
67
+ def run
68
+ @stats[:start_time] = Time.now
69
+
70
+ if @opts[:syslog_channel]
71
+ begin
72
+ require 'syslog_logger'
73
+ @@logger = SyslogLogger.new(@opts[:syslog_channel])
74
+ rescue LoadError
75
+ # SyslogLogger isn't available, so we're just going to use Logger
76
+ end
77
+ end
78
+
79
+ @@logger ||= case @opts[:logger]
80
+ when IO, String; Logger.new(@opts[:logger])
81
+ when Logger; @opts[:logger]
82
+ else; Logger.new(STDERR)
83
+ end
84
+
85
+ begin
86
+ @opts[:queue] = QueueCollection.new(@opts[:path])
87
+ rescue InaccessibleQueuePath => e
88
+ puts "Error: #{e.message}"
89
+ exit 1
90
+ end
91
+ @@logger.level = @opts[:log_level] || Logger::ERROR
92
+
93
+ @@logger.info "Starling STARTUP on #{@opts[:host]}:#{@opts[:port]}"
94
+
95
+ EventMachine.epoll
96
+ EventMachine.set_descriptor_table_size(4096)
97
+ EventMachine.run do
98
+ EventMachine.start_server(@opts[:host], @opts[:port], Handler, @opts)
99
+ end
100
+
101
+ # code here will get executed on shutdown:
102
+ @opts[:queue].close
103
+ end
104
+
105
+ def self.logger
106
+ @@logger
107
+ end
108
+
109
+
110
+ ##
111
+ # Stop accepting new connections and shutdown gracefully.
112
+
113
+ def stop
114
+ EventMachine.stop_event_loop
115
+ end
116
+
117
+ def stats(stat = nil) #:nodoc:
118
+ case stat
119
+ when nil; @stats
120
+ when :connections; 1
121
+ else; @stats[stat]
122
+ end
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,317 @@
1
+ require File.join(File.dirname(__FILE__), 'server')
2
+ require 'erb'
3
+ require 'fileutils'
4
+ require 'optparse'
5
+ require 'yaml'
6
+ require 'fileutils'
7
+
8
+ module StarlingServer
9
+ class Runner
10
+
11
+ attr_accessor :options
12
+ private :options, :options=
13
+
14
+ def self.run
15
+ new
16
+ end
17
+
18
+ def self.shutdown
19
+ @@instance.shutdown
20
+ end
21
+
22
+ def initialize
23
+ @@instance = self
24
+ parse_options
25
+
26
+ @process = ProcessHelper.new(options[:logger], options[:pid_file], options[:user], options[:group])
27
+
28
+ pid = @process.running?
29
+ if pid
30
+ STDERR.puts "There is already a Starling process running (pid #{pid}), exiting."
31
+ exit(1)
32
+ elsif pid.nil?
33
+ STDERR.puts "Cleaning up stale pidfile at #{options[:pid_file]}."
34
+ end
35
+
36
+ start
37
+ end
38
+
39
+ def load_config_file(filename)
40
+ config = YAML.load(ERB.new(File.read(filename)).result)
41
+
42
+ unless config.is_a?(Hash)
43
+ STDERR.puts "Config file does not contain a hash: #{filename}, exiting."
44
+ exit(1)
45
+ end
46
+
47
+ if config['starling'].nil?
48
+ STDERR.puts "Missing starling section in config file: #{filename}, exiting."
49
+ exit(1)
50
+ end
51
+
52
+ config['starling'].each do |key, value|
53
+ # alias some keys
54
+ case key
55
+ when "queue_path" then key = "path"
56
+ when "log_file" then key = "logger"
57
+ end
58
+
59
+ if %w(logger path pid_file).include?(key)
60
+ value = File.expand_path(value)
61
+ end
62
+
63
+ options[key.to_sym] = value
64
+
65
+ if options[:log_level].instance_of?(String)
66
+ options[:log_level] = Logger.const_get(options[:log_level])
67
+ end
68
+ end
69
+ end
70
+
71
+ def parse_options
72
+ self.options = { :host => '127.0.0.1',
73
+ :port => 22122,
74
+ :path => File.join('', 'var', 'spool', 'starling'),
75
+ :log_level => Logger::INFO,
76
+ :daemonize => false,
77
+ :timeout => 0,
78
+ :pid_file => File.join('', 'var', 'run', 'starling.pid') }
79
+
80
+ OptionParser.new do |opts|
81
+ opts.summary_width = 25
82
+
83
+ opts.banner = "Starling (#{StarlingServer::VERSION})\n\n",
84
+ "usage: starling [options...]\n",
85
+ " starling --help\n",
86
+ " starling --version\n"
87
+
88
+ opts.separator ""
89
+ opts.separator "Configuration:"
90
+
91
+ opts.on("-f", "--config FILENAME",
92
+ "Config file (yaml) to load") do |filename|
93
+ load_config_file(filename)
94
+ end
95
+
96
+ opts.on("-q", "--queue_path PATH",
97
+ :REQUIRED,
98
+ "Path to store Starling queue logs", "(default: #{options[:path]})") do |queue_path|
99
+ options[:path] = File.expand_path(queue_path)
100
+ end
101
+
102
+ opts.separator ""; opts.separator "Network:"
103
+
104
+ opts.on("-hHOST", "--host HOST", "Interface on which to listen (default: #{options[:host]})") do |host|
105
+ options[:host] = host
106
+ end
107
+
108
+ opts.on("-pHOST", "--port PORT", Integer, "TCP port on which to listen (default: #{options[:port]})") do |port|
109
+ options[:port] = port
110
+ end
111
+
112
+ opts.separator ""; opts.separator "Process:"
113
+
114
+ opts.on("-d", "Run as a daemon.") do
115
+ options[:daemonize] = true
116
+ end
117
+
118
+ opts.on("-PFILE", "--pid FILENAME", "save PID in FILENAME when using -d option.", "(default: #{options[:pid_file]})") do |pid_file|
119
+ options[:pid_file] = File.expand_path(pid_file)
120
+ end
121
+
122
+ opts.on("-u", "--user USER", "User to run as") do |user|
123
+ options[:user] = user.to_i == 0 ? Etc.getpwnam(user).uid : user.to_i
124
+ end
125
+
126
+ opts.on("-gGROUP", "--group GROUP", "Group to run as") do |group|
127
+ options[:group] = group.to_i == 0 ? Etc.getgrnam(group).gid : group.to_i
128
+ end
129
+
130
+ opts.separator ""; opts.separator "Logging:"
131
+
132
+ opts.on("-L", "--log [FILE]", "Path to print debugging information.") do |log_path|
133
+ options[:logger] = File.expand_path(log_path)
134
+ end
135
+
136
+ begin
137
+ require 'syslog_logger'
138
+
139
+ opts.on("-l", "--syslog CHANNEL", "Write logs to the syslog instead of a log file.") do |channel|
140
+ options[:syslog_channel] = channel
141
+ end
142
+ rescue LoadError
143
+ end
144
+
145
+ opts.on("-v", "Increase logging verbosity (may be used multiple times).") do
146
+ options[:log_level] -= 1
147
+ end
148
+
149
+ opts.on("-t", "--timeout [SECONDS]", Integer,
150
+ "Time in seconds before disconnecting inactive clients (0 to disable).",
151
+ "(default: #{options[:timeout]})") do |timeout|
152
+ options[:timeout] = timeout
153
+ end
154
+
155
+ opts.separator ""; opts.separator "Miscellaneous:"
156
+
157
+ opts.on_tail("-?", "--help", "Display this usage information.") do
158
+ puts "#{opts}\n"
159
+ exit
160
+ end
161
+
162
+ opts.on_tail("-V", "--version", "Print version number and exit.") do
163
+ puts "Starling #{StarlingServer::VERSION}\n\n"
164
+ exit
165
+ end
166
+ end.parse!
167
+ end
168
+
169
+ def start
170
+ drop_privileges
171
+
172
+ @process.daemonize if options[:daemonize]
173
+
174
+ setup_signal_traps
175
+ @process.write_pid_file
176
+
177
+ STDOUT.puts "Starting at #{options[:host]}:#{options[:port]}."
178
+ @server = StarlingServer::Base.new(options)
179
+ @server.run
180
+
181
+ @process.remove_pid_file
182
+ end
183
+
184
+ def drop_privileges
185
+ Process.egid = options[:group] if options[:group]
186
+ Process.euid = options[:user] if options[:user]
187
+ end
188
+
189
+ def shutdown
190
+ begin
191
+ STDOUT.puts "Shutting down."
192
+ StarlingServer::Base.logger.info "Shutting down."
193
+ @server.stop
194
+ rescue Object => e
195
+ STDERR.puts "There was an error shutting down: #{e}"
196
+ exit(70)
197
+ end
198
+ end
199
+
200
+ def setup_signal_traps
201
+ Signal.trap("INT") { shutdown }
202
+ Signal.trap("TERM") { shutdown }
203
+ end
204
+ end
205
+
206
+ class ProcessHelper
207
+
208
+ def initialize(log_file = nil, pid_file = nil, user = nil, group = nil)
209
+ @log_file = log_file
210
+ @pid_file = pid_file
211
+ @user = user
212
+ @group = group
213
+ end
214
+
215
+ def safefork
216
+ begin
217
+ if pid = fork
218
+ return pid
219
+ end
220
+ rescue Errno::EWOULDBLOCK
221
+ sleep 5
222
+ retry
223
+ end
224
+ end
225
+
226
+ def daemonize
227
+ sess_id = detach_from_terminal
228
+ exit if pid = safefork
229
+
230
+ Dir.chdir("/")
231
+ File.umask 0000
232
+
233
+ close_io_handles
234
+ redirect_io
235
+
236
+ return sess_id
237
+ end
238
+
239
+ def detach_from_terminal
240
+ srand
241
+ safefork and exit
242
+
243
+ unless sess_id = Process.setsid
244
+ raise "Couldn't detach from controlling terminal."
245
+ end
246
+
247
+ trap 'SIGHUP', 'IGNORE'
248
+
249
+ sess_id
250
+ end
251
+
252
+ def close_io_handles
253
+ ObjectSpace.each_object(IO) do |io|
254
+ unless [STDIN, STDOUT, STDERR].include?(io)
255
+ begin
256
+ io.close unless io.closed?
257
+ rescue Exception
258
+ end
259
+ end
260
+ end
261
+ end
262
+
263
+ def redirect_io
264
+ begin; STDIN.reopen('/dev/null'); rescue Exception; end
265
+
266
+ if @log_file
267
+ begin
268
+ STDOUT.reopen(@log_file, "a")
269
+ STDOUT.sync = true
270
+ rescue Exception
271
+ begin; STDOUT.reopen('/dev/null'); rescue Exception; end
272
+ end
273
+ else
274
+ begin; STDOUT.reopen('/dev/null'); rescue Exception; end
275
+ end
276
+
277
+ begin; STDERR.reopen(STDOUT); rescue Exception; end
278
+ STDERR.sync = true
279
+ end
280
+
281
+ def rescue_exception
282
+ begin
283
+ yield
284
+ rescue Exception
285
+ end
286
+ end
287
+
288
+ def write_pid_file
289
+ return unless @pid_file
290
+ FileUtils.mkdir_p(File.dirname(@pid_file))
291
+ File.open(@pid_file, "w") { |f| f.write(Process.pid) }
292
+ File.chmod(0644, @pid_file)
293
+ end
294
+
295
+ def remove_pid_file
296
+ return unless @pid_file
297
+ File.unlink(@pid_file) if File.exists?(@pid_file)
298
+ end
299
+
300
+ def running?
301
+ return false unless @pid_file
302
+
303
+ pid = File.read(@pid_file).chomp.to_i rescue nil
304
+ pid = nil if pid == 0
305
+ return false unless pid
306
+
307
+ begin
308
+ Process.kill(0, pid)
309
+ return pid
310
+ rescue Errno::ESRCH
311
+ return nil
312
+ rescue Errno::EPERM
313
+ return pid
314
+ end
315
+ end
316
+ end
317
+ end