twirl 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +17 -0
- data/Gemfile +10 -0
- data/LICENSE.txt +22 -0
- data/README.md +75 -0
- data/Rakefile +1 -0
- data/examples/server.rb +37 -0
- data/lib/twirl.rb +8 -0
- data/lib/twirl/cluster.rb +317 -0
- data/lib/twirl/instrumentation/log_subscriber.rb +35 -0
- data/lib/twirl/instrumentation/statsd.rb +6 -0
- data/lib/twirl/instrumentation/statsd_subscriber.rb +28 -0
- data/lib/twirl/instrumentation/subscriber.rb +60 -0
- data/lib/twirl/instrumenters/memory.rb +26 -0
- data/lib/twirl/instrumenters/noop.rb +9 -0
- data/lib/twirl/item.rb +49 -0
- data/lib/twirl/server.rb +253 -0
- data/lib/twirl/version.rb +3 -0
- data/script/bootstrap +21 -0
- data/script/kestrel +34 -0
- data/script/release +42 -0
- data/script/test +25 -0
- data/script/watch +29 -0
- data/test/cluster_test.rb +469 -0
- data/test/helper.rb +65 -0
- data/test/instrumentation/log_subscriber_test.rb +51 -0
- data/test/instrumentation/statsd_test.rb +173 -0
- data/test/instrumenters/memory_test.rb +22 -0
- data/test/instrumenters/noop_test.rb +16 -0
- data/test/integration/cluster_test.rb +199 -0
- data/test/item_test.rb +43 -0
- data/test/support/fake_udp_socket.rb +27 -0
- data/test/twirl_test.rb +11 -0
- data/twirl.gemspec +23 -0
- metadata +121 -0
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'securerandom'
|
2
|
+
require 'active_support/notifications'
|
3
|
+
require 'active_support/log_subscriber'
|
4
|
+
|
5
|
+
module Twirl
|
6
|
+
module Instrumentation
|
7
|
+
class LogSubscriber < ::ActiveSupport::LogSubscriber
|
8
|
+
def op(event)
|
9
|
+
return unless logger.debug?
|
10
|
+
|
11
|
+
op = event.payload[:op]
|
12
|
+
bytes = event.payload[:bytes]
|
13
|
+
queue_name = event.payload[:queue_name]
|
14
|
+
|
15
|
+
return unless op
|
16
|
+
|
17
|
+
description = "Twirl op(#{op})"
|
18
|
+
details = ""
|
19
|
+
|
20
|
+
if queue_name
|
21
|
+
details += "queue_name=#{queue_name} "
|
22
|
+
end
|
23
|
+
|
24
|
+
if bytes
|
25
|
+
details += "bytes=#{bytes} "
|
26
|
+
end
|
27
|
+
|
28
|
+
name = '%s (%.1fms)' % [description, event.duration]
|
29
|
+
debug " #{color(name, CYAN, true)} [ #{details} ]"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
Instrumentation::LogSubscriber.attach_to :twirl
|
35
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# Note: You should never need to require this file directly if you are using
|
2
|
+
# ActiveSupport::Notifications. Instead, you should require the metriks file
|
3
|
+
# that lives in the same directory as this file. The benefit is that it
|
4
|
+
# subscribes to the correct events and does everything for your.
|
5
|
+
require "twirl/instrumentation/subscriber"
|
6
|
+
require "statsd"
|
7
|
+
|
8
|
+
module Twirl
|
9
|
+
module Instrumentation
|
10
|
+
class StatsdSubscriber < Subscriber
|
11
|
+
class << self
|
12
|
+
attr_accessor :client
|
13
|
+
end
|
14
|
+
|
15
|
+
def update_timer(metric)
|
16
|
+
if self.class.client
|
17
|
+
self.class.client.timing metric, (@duration * 1_000).round
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def update_counter(metric, value = 1)
|
22
|
+
if self.class.client
|
23
|
+
self.class.client.increment metric, value
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module Twirl
|
2
|
+
module Instrumentation
|
3
|
+
class Subscriber
|
4
|
+
# Public: Use this as the subscribed block.
|
5
|
+
def self.call(name, start, ending, transaction_id, payload)
|
6
|
+
new(name, start, ending, transaction_id, payload).update
|
7
|
+
end
|
8
|
+
|
9
|
+
# Private: Initializes a new event processing instance.
|
10
|
+
def initialize(name, start, ending, transaction_id, payload)
|
11
|
+
@name = name
|
12
|
+
@start = start
|
13
|
+
@ending = ending
|
14
|
+
@payload = payload
|
15
|
+
@duration = ending - start
|
16
|
+
@transaction_id = transaction_id
|
17
|
+
end
|
18
|
+
|
19
|
+
# Public: Actually update all the metriks timers for the event.
|
20
|
+
#
|
21
|
+
# Returns nothing.
|
22
|
+
def update
|
23
|
+
op = @payload[:op]
|
24
|
+
bytes = @payload[:bytes]
|
25
|
+
queue_name = @payload[:queue_name]
|
26
|
+
metric_type = @payload[:metric_type] || :timer
|
27
|
+
|
28
|
+
return unless op
|
29
|
+
|
30
|
+
send "update_#{metric_type}", "twirl.op_#{op}"
|
31
|
+
|
32
|
+
if @payload[:retry]
|
33
|
+
update_counter "twirl.retries_op_#{op}"
|
34
|
+
end
|
35
|
+
|
36
|
+
if bytes
|
37
|
+
update_counter "twirl.bytes_op_#{op}"
|
38
|
+
|
39
|
+
if queue_name
|
40
|
+
update_counter "twirl.bytes_queue_#{queue_name}_op_#{op}"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
if queue_name
|
45
|
+
update_timer "twirl.queue_#{queue_name}_op_#{op}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# Internal: Override in subclass.
|
50
|
+
def update_timer(metric)
|
51
|
+
raise NotImplementedError
|
52
|
+
end
|
53
|
+
|
54
|
+
# Internal: Override in subclass.
|
55
|
+
def update_counter(metric, value = 1)
|
56
|
+
raise NotImplementedError
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Twirl
|
2
|
+
module Instrumenters
|
3
|
+
# Instrumentor that is useful for tests as it stores each of the events that
|
4
|
+
# are instrumented.
|
5
|
+
class Memory
|
6
|
+
Event = Struct.new(:name, :payload, :result)
|
7
|
+
|
8
|
+
attr_reader :events
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@events = []
|
12
|
+
end
|
13
|
+
|
14
|
+
def instrument(name, payload = {})
|
15
|
+
result = if block_given?
|
16
|
+
yield payload
|
17
|
+
else
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
|
21
|
+
@events << Event.new(name, payload, result)
|
22
|
+
result
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/lib/twirl/item.rb
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
require "twirl/instrumenters/noop"
|
2
|
+
|
3
|
+
module Twirl
|
4
|
+
class Item
|
5
|
+
# Public: The key of the item.
|
6
|
+
attr_reader :key
|
7
|
+
|
8
|
+
# Public: The value of the item.
|
9
|
+
attr_reader :value
|
10
|
+
|
11
|
+
# Private: The client that popped the item.
|
12
|
+
attr_reader :client
|
13
|
+
|
14
|
+
def initialize(key, value, client, instrumenter = nil)
|
15
|
+
@key = key
|
16
|
+
@value = value
|
17
|
+
@client = client
|
18
|
+
@instrumenter = instrumenter || Instrumenters::Noop
|
19
|
+
end
|
20
|
+
|
21
|
+
# Public: Acknowledge that we are done processing the item.
|
22
|
+
def close
|
23
|
+
@instrumenter.instrument "op.twirl" do |payload|
|
24
|
+
payload[:op] = :item_close
|
25
|
+
payload[:queue_name] = @key
|
26
|
+
|
27
|
+
@client.close @key
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
# Public: Something went wrong processing.
|
32
|
+
def abort
|
33
|
+
@instrumenter.instrument "op.twirl" do |payload|
|
34
|
+
payload[:op] = :item_abort
|
35
|
+
payload[:queue_name] = @key
|
36
|
+
|
37
|
+
@client.abort @key
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def eql?(other)
|
42
|
+
self.class.eql?(other.class) &&
|
43
|
+
@key == other.key &&
|
44
|
+
@value == other.value &&
|
45
|
+
@client == other.client
|
46
|
+
end
|
47
|
+
alias_method :==, :eql?
|
48
|
+
end
|
49
|
+
end
|
data/lib/twirl/server.rb
ADDED
@@ -0,0 +1,253 @@
|
|
1
|
+
require "uri"
|
2
|
+
require "json"
|
3
|
+
require "net/http"
|
4
|
+
require "erb"
|
5
|
+
require "fileutils"
|
6
|
+
|
7
|
+
module Twirl
|
8
|
+
class Server
|
9
|
+
ConfigTemplate = <<_EOC
|
10
|
+
import com.twitter.conversions.storage._
|
11
|
+
import com.twitter.conversions.time._
|
12
|
+
import com.twitter.logging.config._
|
13
|
+
import com.twitter.ostrich.admin.config._
|
14
|
+
import net.lag.kestrel.config._
|
15
|
+
|
16
|
+
new KestrelConfig {
|
17
|
+
listenAddress = "0.0.0.0"
|
18
|
+
memcacheListenPort = <%= @memcache_port %>
|
19
|
+
textListenPort = <%= @text_port %>
|
20
|
+
thriftListenPort = <%= @thrift_port %>
|
21
|
+
|
22
|
+
queuePath = "<%= @queue_path %>"
|
23
|
+
|
24
|
+
clientTimeout = 30.seconds
|
25
|
+
|
26
|
+
expirationTimerFrequency = 1.second
|
27
|
+
|
28
|
+
maxOpenTransactions = 100
|
29
|
+
|
30
|
+
default.defaultJournalSize = 16.megabytes
|
31
|
+
default.maxMemorySize = 128.megabytes
|
32
|
+
default.maxJournalSize = 1.gigabyte
|
33
|
+
|
34
|
+
admin.httpPort = <%= @admin_port %>
|
35
|
+
|
36
|
+
admin.statsNodes = new StatsConfig {
|
37
|
+
reporters = new TimeSeriesCollectorConfig
|
38
|
+
}
|
39
|
+
|
40
|
+
loggers = new LoggerConfig {
|
41
|
+
level = Level.DEBUG
|
42
|
+
handlers = new FileHandlerConfig {
|
43
|
+
filename = "<%= @log_file %>"
|
44
|
+
roll = Policy.Never
|
45
|
+
}
|
46
|
+
}
|
47
|
+
}
|
48
|
+
_EOC
|
49
|
+
|
50
|
+
# Public: The version kestrel will run.
|
51
|
+
attr_reader :version
|
52
|
+
|
53
|
+
# Public: The memcache_port kestrel will run on.
|
54
|
+
attr_reader :memcache_port
|
55
|
+
|
56
|
+
# Public: The thrift_port kestrel will run on.
|
57
|
+
attr_reader :thrift_port
|
58
|
+
|
59
|
+
# Public: The text_port kestrel will run on.
|
60
|
+
attr_reader :text_port
|
61
|
+
|
62
|
+
# Public: The admin_port kestrel will run on.
|
63
|
+
attr_reader :admin_port
|
64
|
+
|
65
|
+
def initialize(dir, options = {})
|
66
|
+
@dir = dir
|
67
|
+
|
68
|
+
@version = options.fetch(:version) { "2.4.1" }
|
69
|
+
@memcache_port = options.fetch(:memcache_port) { 22133 }
|
70
|
+
@thrift_port = options.fetch(:thrift_port) { 2229 }
|
71
|
+
@text_port = options.fetch(:text_port) { 2222 }
|
72
|
+
@admin_port = options.fetch(:admin_port) { 2223 }
|
73
|
+
|
74
|
+
@download_dir = options.fetch(:download_dir) { "/tmp" }
|
75
|
+
@stage = options.fetch(:stage) { "twirl" }
|
76
|
+
|
77
|
+
@remote_zip = "http://robey.github.io/kestrel/download/kestrel-#{@version}.zip"
|
78
|
+
@zip_file = File.join(@download_dir, "kestrel-#{@version}.zip")
|
79
|
+
@unzipped_file = File.join(@download_dir, "kestrel-#{@version}")
|
80
|
+
@jar_file = File.join(@unzipped_file, "kestrel_2.9.2-#{@version}.jar")
|
81
|
+
|
82
|
+
@queue_path = File.join(@dir, "data")
|
83
|
+
@log_path = File.join(@dir, "logs")
|
84
|
+
@config_path = File.join(@dir, "config")
|
85
|
+
|
86
|
+
@log_file = File.join(@log_path, "kestrel.log")
|
87
|
+
@config_file = File.join(@config_path, "#{@stage}.scala")
|
88
|
+
end
|
89
|
+
|
90
|
+
# Public: Downloads, unzips and starts the server.
|
91
|
+
def start
|
92
|
+
ensure_downloaded
|
93
|
+
ensure_unzipped
|
94
|
+
ensure_configured
|
95
|
+
start_server
|
96
|
+
end
|
97
|
+
|
98
|
+
# Public: Stops the server.
|
99
|
+
def stop
|
100
|
+
stop_server
|
101
|
+
end
|
102
|
+
|
103
|
+
# Private: Downloads the file if it has not been downloaded.
|
104
|
+
def ensure_downloaded
|
105
|
+
download unless downloaded?
|
106
|
+
end
|
107
|
+
|
108
|
+
# Private: Returns true or false depending on whether the file has
|
109
|
+
# been downloaded.
|
110
|
+
def downloaded?
|
111
|
+
File.exists?(@zip_file)
|
112
|
+
end
|
113
|
+
|
114
|
+
# Private: Downloads the file.
|
115
|
+
def download
|
116
|
+
uri = URI(@remote_zip)
|
117
|
+
|
118
|
+
Net::HTTP.start(uri.host, uri.port) do |http|
|
119
|
+
request = Net::HTTP::Get.new uri.path
|
120
|
+
|
121
|
+
http.request request do |response|
|
122
|
+
if response.code.to_i == 200
|
123
|
+
downloaded = 0
|
124
|
+
last_percent = 0
|
125
|
+
total = response["content-length"].to_i
|
126
|
+
puts "Downloading #{total} bytes to #{@zip_file}"
|
127
|
+
|
128
|
+
File.open @zip_file, "w" do |io|
|
129
|
+
response.read_body do |chunk|
|
130
|
+
io.write chunk
|
131
|
+
downloaded += chunk.size
|
132
|
+
percent_complete = ((downloaded.to_f / total) * 100).round
|
133
|
+
show_status = percent_complete % 5 == 0 && last_percent != percent_complete
|
134
|
+
|
135
|
+
if show_status
|
136
|
+
last_percent = percent_complete
|
137
|
+
puts "#{downloaded}/#{total}\t#{percent_complete}%"
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
else
|
142
|
+
abort "Could not downloaded kestrel from #{@remote_zip} #{response.inspect}"
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
# Private: Ensures that file is unzipped if downloaded.
|
149
|
+
def ensure_unzipped
|
150
|
+
if downloaded? && !unzipped?
|
151
|
+
unzip
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
# Private: Unzips the file.
|
156
|
+
def unzip
|
157
|
+
Dir.chdir(File.dirname(@zip_file)) do
|
158
|
+
system "unzip", "-o", @zip_file
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
# Private: Returns true or false depending on whether the file has
|
163
|
+
# been unzipped.
|
164
|
+
def unzipped?
|
165
|
+
File.exists?(@unzipped_file)
|
166
|
+
end
|
167
|
+
|
168
|
+
# Private: Ensure directories and configuration files are ready to go.
|
169
|
+
def ensure_configured
|
170
|
+
[
|
171
|
+
@queue_path,
|
172
|
+
@log_path,
|
173
|
+
@config_path,
|
174
|
+
].each do |path|
|
175
|
+
FileUtils.mkdir_p path
|
176
|
+
end
|
177
|
+
|
178
|
+
config_contents = ERB.new(ConfigTemplate).result(binding)
|
179
|
+
File.write @config_file, config_contents
|
180
|
+
end
|
181
|
+
|
182
|
+
# Private: Starts the server. Assumes downloaded and unzipped
|
183
|
+
def start_server
|
184
|
+
puts "Starting server."
|
185
|
+
Dir.chdir(@dir) do
|
186
|
+
system "java -jar #{@jar_file} -f #{@config_file} &"
|
187
|
+
|
188
|
+
loop do
|
189
|
+
break if running?
|
190
|
+
end
|
191
|
+
end
|
192
|
+
puts "Started server."
|
193
|
+
end
|
194
|
+
|
195
|
+
# Private: Stops the server.
|
196
|
+
def stop_server
|
197
|
+
puts "Stopping server."
|
198
|
+
shutdown
|
199
|
+
|
200
|
+
loop do
|
201
|
+
break if stopped?
|
202
|
+
end
|
203
|
+
puts "Stopped server."
|
204
|
+
end
|
205
|
+
|
206
|
+
# Private: Returns true if server is running else false.
|
207
|
+
def running?
|
208
|
+
return "pong" == ping
|
209
|
+
rescue => exception
|
210
|
+
$stderr.puts exception.inspect
|
211
|
+
false
|
212
|
+
end
|
213
|
+
|
214
|
+
# Private: Returns true if server is stopped else false.
|
215
|
+
def stopped?
|
216
|
+
return !running?
|
217
|
+
end
|
218
|
+
|
219
|
+
# Private: Prints out the status of the server.
|
220
|
+
def status
|
221
|
+
if running?
|
222
|
+
:running
|
223
|
+
else
|
224
|
+
:stopped
|
225
|
+
end
|
226
|
+
rescue => exception
|
227
|
+
:unknown
|
228
|
+
end
|
229
|
+
|
230
|
+
# Private: Pings the kestrel server.
|
231
|
+
def ping
|
232
|
+
get_response("ping")["response"]
|
233
|
+
end
|
234
|
+
|
235
|
+
# Private: Shutsdown the kestrel server.
|
236
|
+
def shutdown
|
237
|
+
h = get_response("shutdown")
|
238
|
+
return h["response"] == "ok"
|
239
|
+
rescue => exception
|
240
|
+
puts "Failed to shutdown: #{exception.inspect}"
|
241
|
+
false
|
242
|
+
end
|
243
|
+
|
244
|
+
# Private: Allows requesting things from the kestrel admin.
|
245
|
+
def get_response(path)
|
246
|
+
uri = URI.parse("http://localhost:#{@admin_port}/#{path}")
|
247
|
+
response = Net::HTTP.get_response(uri)
|
248
|
+
JSON.parse(response.body)
|
249
|
+
rescue => exception
|
250
|
+
{}
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|