twirl 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,35 @@
1
+ require 'securerandom'
2
+ require 'active_support/notifications'
3
+ require 'active_support/log_subscriber'
4
+
5
+ module Twirl
6
+ module Instrumentation
7
+ class LogSubscriber < ::ActiveSupport::LogSubscriber
8
+ def op(event)
9
+ return unless logger.debug?
10
+
11
+ op = event.payload[:op]
12
+ bytes = event.payload[:bytes]
13
+ queue_name = event.payload[:queue_name]
14
+
15
+ return unless op
16
+
17
+ description = "Twirl op(#{op})"
18
+ details = ""
19
+
20
+ if queue_name
21
+ details += "queue_name=#{queue_name} "
22
+ end
23
+
24
+ if bytes
25
+ details += "bytes=#{bytes} "
26
+ end
27
+
28
+ name = '%s (%.1fms)' % [description, event.duration]
29
+ debug " #{color(name, CYAN, true)} [ #{details} ]"
30
+ end
31
+ end
32
+ end
33
+
34
+ Instrumentation::LogSubscriber.attach_to :twirl
35
+ end
@@ -0,0 +1,6 @@
1
+ require "securerandom"
2
+ require "active_support/notifications"
3
+ require "twirl/instrumentation/statsd_subscriber"
4
+
5
+ ActiveSupport::Notifications.subscribe 'op.twirl',
6
+ Twirl::Instrumentation::StatsdSubscriber
@@ -0,0 +1,28 @@
1
+ # Note: You should never need to require this file directly if you are using
2
+ # ActiveSupport::Notifications. Instead, you should require the metriks file
3
+ # that lives in the same directory as this file. The benefit is that it
4
+ # subscribes to the correct events and does everything for your.
5
+ require "twirl/instrumentation/subscriber"
6
+ require "statsd"
7
+
8
+ module Twirl
9
+ module Instrumentation
10
+ class StatsdSubscriber < Subscriber
11
+ class << self
12
+ attr_accessor :client
13
+ end
14
+
15
+ def update_timer(metric)
16
+ if self.class.client
17
+ self.class.client.timing metric, (@duration * 1_000).round
18
+ end
19
+ end
20
+
21
+ def update_counter(metric, value = 1)
22
+ if self.class.client
23
+ self.class.client.increment metric, value
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,60 @@
1
+ module Twirl
2
+ module Instrumentation
3
+ class Subscriber
4
+ # Public: Use this as the subscribed block.
5
+ def self.call(name, start, ending, transaction_id, payload)
6
+ new(name, start, ending, transaction_id, payload).update
7
+ end
8
+
9
+ # Private: Initializes a new event processing instance.
10
+ def initialize(name, start, ending, transaction_id, payload)
11
+ @name = name
12
+ @start = start
13
+ @ending = ending
14
+ @payload = payload
15
+ @duration = ending - start
16
+ @transaction_id = transaction_id
17
+ end
18
+
19
+ # Public: Actually update all the metriks timers for the event.
20
+ #
21
+ # Returns nothing.
22
+ def update
23
+ op = @payload[:op]
24
+ bytes = @payload[:bytes]
25
+ queue_name = @payload[:queue_name]
26
+ metric_type = @payload[:metric_type] || :timer
27
+
28
+ return unless op
29
+
30
+ send "update_#{metric_type}", "twirl.op_#{op}"
31
+
32
+ if @payload[:retry]
33
+ update_counter "twirl.retries_op_#{op}"
34
+ end
35
+
36
+ if bytes
37
+ update_counter "twirl.bytes_op_#{op}"
38
+
39
+ if queue_name
40
+ update_counter "twirl.bytes_queue_#{queue_name}_op_#{op}"
41
+ end
42
+ end
43
+
44
+ if queue_name
45
+ update_timer "twirl.queue_#{queue_name}_op_#{op}"
46
+ end
47
+ end
48
+
49
+ # Internal: Override in subclass.
50
+ def update_timer(metric)
51
+ raise NotImplementedError
52
+ end
53
+
54
+ # Internal: Override in subclass.
55
+ def update_counter(metric, value = 1)
56
+ raise NotImplementedError
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,26 @@
1
+ module Twirl
2
+ module Instrumenters
3
+ # Instrumentor that is useful for tests as it stores each of the events that
4
+ # are instrumented.
5
+ class Memory
6
+ Event = Struct.new(:name, :payload, :result)
7
+
8
+ attr_reader :events
9
+
10
+ def initialize
11
+ @events = []
12
+ end
13
+
14
+ def instrument(name, payload = {})
15
+ result = if block_given?
16
+ yield payload
17
+ else
18
+ nil
19
+ end
20
+
21
+ @events << Event.new(name, payload, result)
22
+ result
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,9 @@
1
+ module Twirl
2
+ module Instrumenters
3
+ class Noop
4
+ def self.instrument(name, payload = {})
5
+ yield payload if block_given?
6
+ end
7
+ end
8
+ end
9
+ end
data/lib/twirl/item.rb ADDED
@@ -0,0 +1,49 @@
1
+ require "twirl/instrumenters/noop"
2
+
3
+ module Twirl
4
+ class Item
5
+ # Public: The key of the item.
6
+ attr_reader :key
7
+
8
+ # Public: The value of the item.
9
+ attr_reader :value
10
+
11
+ # Private: The client that popped the item.
12
+ attr_reader :client
13
+
14
+ def initialize(key, value, client, instrumenter = nil)
15
+ @key = key
16
+ @value = value
17
+ @client = client
18
+ @instrumenter = instrumenter || Instrumenters::Noop
19
+ end
20
+
21
+ # Public: Acknowledge that we are done processing the item.
22
+ def close
23
+ @instrumenter.instrument "op.twirl" do |payload|
24
+ payload[:op] = :item_close
25
+ payload[:queue_name] = @key
26
+
27
+ @client.close @key
28
+ end
29
+ end
30
+
31
+ # Public: Something went wrong processing.
32
+ def abort
33
+ @instrumenter.instrument "op.twirl" do |payload|
34
+ payload[:op] = :item_abort
35
+ payload[:queue_name] = @key
36
+
37
+ @client.abort @key
38
+ end
39
+ end
40
+
41
+ def eql?(other)
42
+ self.class.eql?(other.class) &&
43
+ @key == other.key &&
44
+ @value == other.value &&
45
+ @client == other.client
46
+ end
47
+ alias_method :==, :eql?
48
+ end
49
+ end
@@ -0,0 +1,253 @@
1
+ require "uri"
2
+ require "json"
3
+ require "net/http"
4
+ require "erb"
5
+ require "fileutils"
6
+
7
+ module Twirl
8
+ class Server
9
+ ConfigTemplate = <<_EOC
10
+ import com.twitter.conversions.storage._
11
+ import com.twitter.conversions.time._
12
+ import com.twitter.logging.config._
13
+ import com.twitter.ostrich.admin.config._
14
+ import net.lag.kestrel.config._
15
+
16
+ new KestrelConfig {
17
+ listenAddress = "0.0.0.0"
18
+ memcacheListenPort = <%= @memcache_port %>
19
+ textListenPort = <%= @text_port %>
20
+ thriftListenPort = <%= @thrift_port %>
21
+
22
+ queuePath = "<%= @queue_path %>"
23
+
24
+ clientTimeout = 30.seconds
25
+
26
+ expirationTimerFrequency = 1.second
27
+
28
+ maxOpenTransactions = 100
29
+
30
+ default.defaultJournalSize = 16.megabytes
31
+ default.maxMemorySize = 128.megabytes
32
+ default.maxJournalSize = 1.gigabyte
33
+
34
+ admin.httpPort = <%= @admin_port %>
35
+
36
+ admin.statsNodes = new StatsConfig {
37
+ reporters = new TimeSeriesCollectorConfig
38
+ }
39
+
40
+ loggers = new LoggerConfig {
41
+ level = Level.DEBUG
42
+ handlers = new FileHandlerConfig {
43
+ filename = "<%= @log_file %>"
44
+ roll = Policy.Never
45
+ }
46
+ }
47
+ }
48
+ _EOC
49
+
50
+ # Public: The version kestrel will run.
51
+ attr_reader :version
52
+
53
+ # Public: The memcache_port kestrel will run on.
54
+ attr_reader :memcache_port
55
+
56
+ # Public: The thrift_port kestrel will run on.
57
+ attr_reader :thrift_port
58
+
59
+ # Public: The text_port kestrel will run on.
60
+ attr_reader :text_port
61
+
62
+ # Public: The admin_port kestrel will run on.
63
+ attr_reader :admin_port
64
+
65
+ def initialize(dir, options = {})
66
+ @dir = dir
67
+
68
+ @version = options.fetch(:version) { "2.4.1" }
69
+ @memcache_port = options.fetch(:memcache_port) { 22133 }
70
+ @thrift_port = options.fetch(:thrift_port) { 2229 }
71
+ @text_port = options.fetch(:text_port) { 2222 }
72
+ @admin_port = options.fetch(:admin_port) { 2223 }
73
+
74
+ @download_dir = options.fetch(:download_dir) { "/tmp" }
75
+ @stage = options.fetch(:stage) { "twirl" }
76
+
77
+ @remote_zip = "http://robey.github.io/kestrel/download/kestrel-#{@version}.zip"
78
+ @zip_file = File.join(@download_dir, "kestrel-#{@version}.zip")
79
+ @unzipped_file = File.join(@download_dir, "kestrel-#{@version}")
80
+ @jar_file = File.join(@unzipped_file, "kestrel_2.9.2-#{@version}.jar")
81
+
82
+ @queue_path = File.join(@dir, "data")
83
+ @log_path = File.join(@dir, "logs")
84
+ @config_path = File.join(@dir, "config")
85
+
86
+ @log_file = File.join(@log_path, "kestrel.log")
87
+ @config_file = File.join(@config_path, "#{@stage}.scala")
88
+ end
89
+
90
+ # Public: Downloads, unzips and starts the server.
91
+ def start
92
+ ensure_downloaded
93
+ ensure_unzipped
94
+ ensure_configured
95
+ start_server
96
+ end
97
+
98
+ # Public: Stops the server.
99
+ def stop
100
+ stop_server
101
+ end
102
+
103
+ # Private: Downloads the file if it has not been downloaded.
104
+ def ensure_downloaded
105
+ download unless downloaded?
106
+ end
107
+
108
+ # Private: Returns true or false depending on whether the file has
109
+ # been downloaded.
110
+ def downloaded?
111
+ File.exists?(@zip_file)
112
+ end
113
+
114
+ # Private: Downloads the file.
115
+ def download
116
+ uri = URI(@remote_zip)
117
+
118
+ Net::HTTP.start(uri.host, uri.port) do |http|
119
+ request = Net::HTTP::Get.new uri.path
120
+
121
+ http.request request do |response|
122
+ if response.code.to_i == 200
123
+ downloaded = 0
124
+ last_percent = 0
125
+ total = response["content-length"].to_i
126
+ puts "Downloading #{total} bytes to #{@zip_file}"
127
+
128
+ File.open @zip_file, "w" do |io|
129
+ response.read_body do |chunk|
130
+ io.write chunk
131
+ downloaded += chunk.size
132
+ percent_complete = ((downloaded.to_f / total) * 100).round
133
+ show_status = percent_complete % 5 == 0 && last_percent != percent_complete
134
+
135
+ if show_status
136
+ last_percent = percent_complete
137
+ puts "#{downloaded}/#{total}\t#{percent_complete}%"
138
+ end
139
+ end
140
+ end
141
+ else
142
+ abort "Could not downloaded kestrel from #{@remote_zip} #{response.inspect}"
143
+ end
144
+ end
145
+ end
146
+ end
147
+
148
+ # Private: Ensures that file is unzipped if downloaded.
149
+ def ensure_unzipped
150
+ if downloaded? && !unzipped?
151
+ unzip
152
+ end
153
+ end
154
+
155
+ # Private: Unzips the file.
156
+ def unzip
157
+ Dir.chdir(File.dirname(@zip_file)) do
158
+ system "unzip", "-o", @zip_file
159
+ end
160
+ end
161
+
162
+ # Private: Returns true or false depending on whether the file has
163
+ # been unzipped.
164
+ def unzipped?
165
+ File.exists?(@unzipped_file)
166
+ end
167
+
168
+ # Private: Ensure directories and configuration files are ready to go.
169
+ def ensure_configured
170
+ [
171
+ @queue_path,
172
+ @log_path,
173
+ @config_path,
174
+ ].each do |path|
175
+ FileUtils.mkdir_p path
176
+ end
177
+
178
+ config_contents = ERB.new(ConfigTemplate).result(binding)
179
+ File.write @config_file, config_contents
180
+ end
181
+
182
+ # Private: Starts the server. Assumes downloaded and unzipped
183
+ def start_server
184
+ puts "Starting server."
185
+ Dir.chdir(@dir) do
186
+ system "java -jar #{@jar_file} -f #{@config_file} &"
187
+
188
+ loop do
189
+ break if running?
190
+ end
191
+ end
192
+ puts "Started server."
193
+ end
194
+
195
+ # Private: Stops the server.
196
+ def stop_server
197
+ puts "Stopping server."
198
+ shutdown
199
+
200
+ loop do
201
+ break if stopped?
202
+ end
203
+ puts "Stopped server."
204
+ end
205
+
206
+ # Private: Returns true if server is running else false.
207
+ def running?
208
+ return "pong" == ping
209
+ rescue => exception
210
+ $stderr.puts exception.inspect
211
+ false
212
+ end
213
+
214
+ # Private: Returns true if server is stopped else false.
215
+ def stopped?
216
+ return !running?
217
+ end
218
+
219
+ # Private: Prints out the status of the server.
220
+ def status
221
+ if running?
222
+ :running
223
+ else
224
+ :stopped
225
+ end
226
+ rescue => exception
227
+ :unknown
228
+ end
229
+
230
+ # Private: Pings the kestrel server.
231
+ def ping
232
+ get_response("ping")["response"]
233
+ end
234
+
235
+ # Private: Shutsdown the kestrel server.
236
+ def shutdown
237
+ h = get_response("shutdown")
238
+ return h["response"] == "ok"
239
+ rescue => exception
240
+ puts "Failed to shutdown: #{exception.inspect}"
241
+ false
242
+ end
243
+
244
+ # Private: Allows requesting things from the kestrel admin.
245
+ def get_response(path)
246
+ uri = URI.parse("http://localhost:#{@admin_port}/#{path}")
247
+ response = Net::HTTP.get_response(uri)
248
+ JSON.parse(response.body)
249
+ rescue => exception
250
+ {}
251
+ end
252
+ end
253
+ end