starling-starling 0.9.7.9

Sign up to get free protection for your applications and to get access to all the features.
data/lib/starling.rb ADDED
@@ -0,0 +1,104 @@
1
+ require 'memcache'
2
+
3
+ class Starling < MemCache
4
+
5
+ WAIT_TIME = 0.25
6
+
7
+ ##
8
+ # fetch an item from a queue.
9
+
10
+ def get(*args)
11
+ loop do
12
+ response = super(*args)
13
+ return response unless response.nil?
14
+ sleep WAIT_TIME
15
+ end
16
+ end
17
+
18
+ ##
19
+ # insert +value+ into +queue+.
20
+ #
21
+ # +expiry+ is expressed as a UNIX timestamp
22
+ #
23
+ # If +raw+ is true, +value+ will not be Marshalled. If +raw+ = :yaml, +value+
24
+ # will be serialized with YAML, instead.
25
+
26
+ def set(queue, value, expiry = 0, raw = false)
27
+ retries = 0
28
+ begin
29
+ if raw == :yaml
30
+ value = YAML.dump(value)
31
+ raw = true
32
+ end
33
+
34
+ super(queue, value, expiry, raw)
35
+ rescue MemCache::MemCacheError => e
36
+ retries += 1
37
+ sleep WAIT_TIME
38
+ retry unless retries > 3
39
+ raise e
40
+ end
41
+ end
42
+
43
+ ##
44
+ # returns the number of items in +queue+. If +queue+ is +:all+, a hash of all
45
+ # queue sizes will be returned.
46
+
47
+ def sizeof(queue, statistics = nil)
48
+ statistics ||= stats
49
+
50
+ if queue == :all
51
+ queue_sizes = {}
52
+ available_queues(statistics).each do |queue|
53
+ queue_sizes[queue] = sizeof(queue, statistics)
54
+ end
55
+ return queue_sizes
56
+ end
57
+
58
+ statistics.inject(0) { |m,(k,v)| m + v["queue_#{queue}_items"].to_i }
59
+ end
60
+
61
+ ##
62
+ # returns a list of available (currently allocated) queues.
63
+
64
+ def available_queues(statistics = nil)
65
+ statistics ||= stats
66
+
67
+ statistics.map { |k,v|
68
+ v.keys
69
+ }.flatten.uniq.grep(/^queue_(.*)_items/).map { |v|
70
+ v.gsub(/^queue_/, '').gsub(/_items$/, '')
71
+ }.reject { |v|
72
+ v =~ /_total$/ || v =~ /_expired$/
73
+ }
74
+ end
75
+
76
+ ##
77
+ # iterator to flush +queue+. Each element will be passed to the provided
78
+ # +block+
79
+
80
+ def flush(queue)
81
+ sizeof(queue).times do
82
+ v = get(queue)
83
+ yield v if block_given?
84
+ end
85
+ end
86
+
87
+ private
88
+
89
+ def get_server_for_key(key)
90
+ raise ArgumentError, "illegal character in key #{key.inspect}" if key =~ /\s/
91
+ raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
92
+ raise MemCacheError, "No servers available" if @servers.empty?
93
+
94
+ bukkits = @buckets.dup
95
+ bukkits.nitems.times do |try|
96
+ n = rand(bukkits.nitems)
97
+ server = bukkits[n]
98
+ return server if server.alive?
99
+ bukkits.delete_at(n)
100
+ end
101
+
102
+ raise MemCacheError, "No servers available (all dead)"
103
+ end
104
+ end
@@ -0,0 +1,205 @@
1
+ $:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
2
+
3
+ require 'rubygems'
4
+ require 'fileutils'
5
+ require 'memcache'
6
+ require 'digest/md5'
7
+
8
+ require 'starling/server'
9
+
10
+ class StarlingServer::PersistentQueue
11
+ remove_const :SOFT_LOG_MAX_SIZE
12
+ SOFT_LOG_MAX_SIZE = 16 * 1024 # 16 KB
13
+ end
14
+
15
+ def safely_fork(&block)
16
+ # anti-race juice:
17
+ blocking = true
18
+ Signal.trap("USR1") { blocking = false }
19
+
20
+ pid = Process.fork(&block)
21
+
22
+ while blocking
23
+ sleep 0.1
24
+ end
25
+
26
+ pid
27
+ end
28
+
29
+ describe "StarlingServer" do
30
+ before do
31
+ @tmp_path = File.join(File.dirname(__FILE__), "tmp")
32
+
33
+ begin
34
+ Dir::mkdir(@tmp_path)
35
+ rescue Errno::EEXIST
36
+ end
37
+
38
+ @server_pid = safely_fork do
39
+ server = StarlingServer::Base.new(:host => '127.0.0.1',
40
+ :port => 22133,
41
+ :path => @tmp_path,
42
+ :logger => Logger.new(STDERR),
43
+ :log_level => Logger::FATAL)
44
+ Signal.trap("INT") { server.stop }
45
+ Process.kill("USR1", Process.ppid)
46
+ server.run
47
+ end
48
+
49
+ @client = MemCache.new('127.0.0.1:22133')
50
+ end
51
+
52
+ it "should test if temp_path exists and is writeable" do
53
+ File.exist?(@tmp_path).should be_true
54
+ File.directory?(@tmp_path).should be_true
55
+ File.writable?(@tmp_path).should be_true
56
+ end
57
+
58
+ it "should set and get" do
59
+ v = rand((2**32)-1)
60
+ @client.get('test_set_and_get_one_entry').should be_nil
61
+ @client.set('test_set_and_get_one_entry', v)
62
+ @client.get('test_set_and_get_one_entry').should eql(v)
63
+ end
64
+
65
+
66
+ it "should expire entries" do
67
+ v = rand((2**32)-1)
68
+ @client.get('test_set_with_expiry').should be_nil
69
+ now = Time.now.to_i
70
+ @client.set('test_set_with_expiry', v + 2, now)
71
+ @client.set('test_set_with_expiry', v)
72
+ sleep(1.0)
73
+ @client.get('test_set_with_expiry').should eql(v)
74
+ end
75
+
76
+ it "should have age stat" do
77
+ now = Time.now.to_i
78
+ @client.set('test_age', 'nibbler')
79
+ sleep(1.0)
80
+ @client.get('test_age').should eql('nibbler')
81
+
82
+ stats = @client.stats['127.0.0.1:22133']
83
+ stats.has_key?('queue_test_age_age').should be_true
84
+ (stats['queue_test_age_age'] >= 1000).should be_true
85
+ end
86
+
87
+ it "should rotate log" do
88
+ log_rotation_path = File.join(@tmp_path, 'test_log_rotation')
89
+
90
+ Dir.glob("#{log_rotation_path}*").each do |file|
91
+ File.unlink(file) rescue nil
92
+ end
93
+ @client.get('test_log_rotation').should be_nil
94
+
95
+ v = 'x' * 8192
96
+
97
+ @client.set('test_log_rotation', v)
98
+ File.size(log_rotation_path).should eql(8207)
99
+ @client.get('test_log_rotation')
100
+
101
+ @client.get('test_log_rotation').should be_nil
102
+
103
+ @client.set('test_log_rotation', v)
104
+ @client.get('test_log_rotation').should eql(v)
105
+
106
+ File.size(log_rotation_path).should eql(1)
107
+ # rotated log should be erased after a successful roll.
108
+ Dir.glob("#{log_rotation_path}*").size.should eql(1)
109
+ end
110
+
111
+ it "should output statistics per server" do
112
+ stats = @client.stats
113
+ assert_kind_of Hash, stats
114
+ assert stats.has_key?('127.0.0.1:22133')
115
+
116
+ server_stats = stats['127.0.0.1:22133']
117
+
118
+ basic_stats = %w( bytes pid time limit_maxbytes cmd_get version
119
+ bytes_written cmd_set get_misses total_connections
120
+ curr_connections curr_items uptime get_hits total_items
121
+ rusage_system rusage_user bytes_read )
122
+
123
+ basic_stats.each do |stat|
124
+ server_stats.has_key?(stat).should be_true
125
+ end
126
+ end
127
+
128
+ it "should return valid response with unkown command" do
129
+ response = @client.add('blah', 1)
130
+ response.should eql("CLIENT_ERROR bad command line format\r\n")
131
+ end
132
+
133
+ it "should disconnect and reconnect again" do
134
+ v = rand(2**32-1)
135
+ @client.set('test_that_disconnecting_and_reconnecting_works', v)
136
+ @client.reset
137
+ @client.get('test_that_disconnecting_and_reconnecting_works').should eql(v)
138
+ end
139
+
140
+ it "should use epoll on linux" do
141
+ # this may take a few seconds.
142
+ # the point is to make sure that we're using epoll on Linux, so we can
143
+ # handle more than 1024 connections.
144
+
145
+ unless IO::popen("uname").read.chomp == "Linux"
146
+ raise "(Skipping epoll test: not on Linux)"
147
+ skip = true
148
+ end
149
+ fd_limit = IO::popen("bash -c 'ulimit -n'").read.chomp.to_i
150
+ unless fd_limit > 1024
151
+ raise "(Skipping epoll test: 'ulimit -n' = #{fd_limit}, need > 1024)"
152
+ skip = true
153
+ end
154
+
155
+ unless skip
156
+ v = rand(2**32 - 1)
157
+ @client.set('test_epoll', v)
158
+
159
+ # we can't open 1024 connections to memcache from within this process,
160
+ # because we will hit ruby's 1024 fd limit ourselves!
161
+ pid1 = safely_fork do
162
+ unused_sockets = []
163
+ 600.times do
164
+ unused_sockets << TCPSocket.new("127.0.0.1", 22133)
165
+ end
166
+ Process.kill("USR1", Process.ppid)
167
+ sleep 90
168
+ end
169
+ pid2 = safely_fork do
170
+ unused_sockets = []
171
+ 600.times do
172
+ unused_sockets << TCPSocket.new("127.0.0.1", 22133)
173
+ end
174
+ Process.kill("USR1", Process.ppid)
175
+ sleep 90
176
+ end
177
+
178
+ begin
179
+ client = MemCache.new('127.0.0.1:22133')
180
+ client.get('test_epoll').should eql(v)
181
+ ensure
182
+ Process.kill("TERM", pid1)
183
+ Process.kill("TERM", pid2)
184
+ end
185
+ end
186
+ end
187
+
188
+ it "should raise error if queue collection is an invalid path" do
189
+ invalid_path = nil
190
+ while invalid_path.nil? || File.exist?(invalid_path)
191
+ invalid_path = File.join('/', Digest::MD5.hexdigest(rand(2**32-1).to_s)[0,8])
192
+ end
193
+
194
+ lambda {
195
+ StarlingServer::QueueCollection.new(invalid_path)
196
+ }.should raise_error(StarlingServer::InaccessibleQueuePath)
197
+ end
198
+
199
+ after do
200
+ Process.kill("INT", @server_pid)
201
+ Process.wait(@server_pid)
202
+ @client.reset
203
+ FileUtils.rm(Dir.glob(File.join(@tmp_path, '*')))
204
+ end
205
+ end
metadata ADDED
@@ -0,0 +1,108 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: starling-starling
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.9.7.9
5
+ platform: ruby
6
+ authors:
7
+ - Blaine Cook
8
+ - Chris Wanstrath
9
+ - Britt Selvitelle
10
+ - Glenn Rempe
11
+ - Abdul-Rahman Advany
12
+ autorequire:
13
+ bindir: bin
14
+ cert_chain: []
15
+
16
+ date: 2008-07-08 00:00:00 -07:00
17
+ default_executable:
18
+ dependencies:
19
+ - !ruby/object:Gem::Dependency
20
+ name: memcache-client
21
+ version_requirement:
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: "0"
27
+ version:
28
+ - !ruby/object:Gem::Dependency
29
+ name: SyslogLogger
30
+ version_requirement:
31
+ version_requirements: !ruby/object:Gem::Requirement
32
+ requirements:
33
+ - - ">="
34
+ - !ruby/object:Gem::Version
35
+ version: "0"
36
+ version:
37
+ - !ruby/object:Gem::Dependency
38
+ name: eventmachine
39
+ version_requirement:
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - ">="
43
+ - !ruby/object:Gem::Version
44
+ version: 0.12.0
45
+ version:
46
+ description: Starling is a lightweight, transactional, distributed queue server
47
+ email:
48
+ - blaine@twitter.com
49
+ - chris@ozmm.org
50
+ - abdulrahman@advany.com
51
+ executables:
52
+ - starling
53
+ - starling_top
54
+ extensions: []
55
+
56
+ extra_rdoc_files:
57
+ - README.rdoc
58
+ - CHANGELOG
59
+ - LICENSE
60
+ files:
61
+ - README.rdoc
62
+ - LICENSE
63
+ - CHANGELOG
64
+ - Rakefile
65
+ - lib/starling/handler.rb
66
+ - lib/starling/persistent_queue.rb
67
+ - lib/starling/queue_collection.rb
68
+ - lib/starling/server_runner.rb
69
+ - lib/starling/server.rb
70
+ - lib/starling.rb
71
+ - etc/starling.redhat
72
+ - etc/starling.ubuntu
73
+ has_rdoc: true
74
+ homepage: http://github.com/starling/starling/
75
+ post_install_message:
76
+ rdoc_options:
77
+ - --quiet
78
+ - --title
79
+ - starling documentation
80
+ - --opname
81
+ - index.html
82
+ - --line-numbers
83
+ - --main
84
+ - README.rdoc
85
+ - --inline-source
86
+ require_paths:
87
+ - lib
88
+ required_ruby_version: !ruby/object:Gem::Requirement
89
+ requirements:
90
+ - - ">="
91
+ - !ruby/object:Gem::Version
92
+ version: "0"
93
+ version:
94
+ required_rubygems_version: !ruby/object:Gem::Requirement
95
+ requirements:
96
+ - - ">="
97
+ - !ruby/object:Gem::Version
98
+ version: "0"
99
+ version:
100
+ requirements: []
101
+
102
+ rubyforge_project:
103
+ rubygems_version: 1.2.0
104
+ signing_key:
105
+ specification_version: 2
106
+ summary: Starling is a lightweight, transactional, distributed queue server
107
+ test_files:
108
+ - test/test_starling_server.rb