squash_repeater 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +128 -0
- data/Rakefile +12 -0
- data/lib/generators/squash_repeater/install_generator.rb +16 -0
- data/lib/generators/templates/squash_repeater_initializer.rb +30 -0
- data/lib/squash_repeater/configure/squash.rb +37 -0
- data/lib/squash_repeater/configure.rb +76 -0
- data/lib/squash_repeater/exception_queue.rb +106 -0
- data/lib/squash_repeater/squash_ruby.rb +19 -0
- data/lib/squash_repeater/version.rb +3 -0
- data/lib/squash_repeater.rb +14 -0
- data/share/upstart/backburner-manager.conf +43 -0
- data/share/upstart/backburner-worker.conf +28 -0
- data/spec/spec_helper.rb +11 -0
- data/spec/squash_repeater_configure_squash.rb +22 -0
- data/spec/squash_repeater_spec.rb +85 -0
- data/spec/squash_ruby_spec.rb +27 -0
- data/squash_repeater-ruby.gemspec +31 -0
- metadata +181 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 219ecac4dc1a24819c44deee687c56d0cadbe2b2
|
4
|
+
data.tar.gz: cf636f9dd0f10862986aa4f09041fd16705f6f19
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c93756e6d025867532c462cc38fb74c46584ffca9ea47cba2f113697e2ab39d520fc0729dc503990fa94affe0c41b1b90cab4099509f26c487150b74ab0811df
|
7
|
+
data.tar.gz: cef8c48c93aea8e1e1136eb93854aff7a790e8717b2bb5ad508828be3720bb749d3e42a8937bd066724a799bf6181485c4ed57eeebfbf0262fe8866bf61c8ef5
|
data/.gitignore
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
# Specify your gem's dependencies in squash_repeater-ruby.gemspec
|
4
|
+
gemspec
|
5
|
+
|
6
|
+
# Latest beaneater has an important fix (see https://github.com/powershop/squash_repeater/issues/2):
|
7
|
+
# You may need to add this to your including library until a new gem of beaneater is released:
|
8
|
+
#gem 'beaneater', :github => 'beanstalkd/beaneater'
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2015 Powershop
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,128 @@
|
|
1
|
+
# SquashRepeater
|
2
|
+
|
3
|
+
One difficulty with Squash is that whenever any exception occurs, it contacts the
|
4
|
+
Squash server to upload details of that failure, which means the code blocks until
|
5
|
+
contact is made or the connection times-out.
|
6
|
+
|
7
|
+
In many cases, an exception is a failure, and you probably don't want to be nice
|
8
|
+
about it, but for user-facing app's, you're degrading the user-experience even
|
9
|
+
further. A quick-response "failure" message to the user is still many times better
|
10
|
+
than waiting many seconds (or more).
|
11
|
+
|
12
|
+
On top of that, almost all exceptions are valuable, in that they have captured a
|
13
|
+
failure-mode that you weren't previously aware of. If the server is down, then the
|
14
|
+
chances are that data is lost.
|
15
|
+
|
16
|
+
Squash Repeater uses a low-overhead queueing service to capture any exceptions and
|
17
|
+
return control as quickly as possible, after which an independent "worker" process
|
18
|
+
will attempt to send the queued exception reports to the Squash server.
|
19
|
+
This should mean:
|
20
|
+
- the local Squash client code can return far more quickly
|
21
|
+
- any exception reports that fail to be accepted by the Squash server aren't lost
|
22
|
+
(or dropped from the queue)
|
23
|
+
- we can "distribute" part of the Squash service throughout all the client servers
|
24
|
+
|
25
|
+
It doesn't solve things like
|
26
|
+
- multi-homed / distributed Squash server
|
27
|
+
|
28
|
+
## Installation
|
29
|
+
|
30
|
+
Add this line to your application's Gemfile:
|
31
|
+
|
32
|
+
```ruby
|
33
|
+
gem 'squash_repeater', github: 'powershop/squash_repeater'
|
34
|
+
```
|
35
|
+
|
36
|
+
And then execute:
|
37
|
+
|
38
|
+
$ bundle
|
39
|
+
|
40
|
+
Or install it yourself as:
|
41
|
+
|
42
|
+
$ gem install squash_repeater
|
43
|
+
|
44
|
+
If you're using Rails, you can install an initialiser template with:
|
45
|
+
|
46
|
+
$ bundle exec rails generate squash_repeater:install
|
47
|
+
|
48
|
+
## Install `beanstalkd`
|
49
|
+
|
50
|
+
- Configure it to listen on `localhost` network
|
51
|
+
- Configure it to auto-start on system boot
|
52
|
+
- Give it a persistent store for queue data (needed in-order to not lose data)
|
53
|
+
|
54
|
+
## Configure `backburner`
|
55
|
+
|
56
|
+
`backburner` is the Gem used to interact with the `beanstalk` queue.
|
57
|
+
It works in two parts: client libraries that put data on the queue, and background-worker jobs that process the data on
|
58
|
+
queue.
|
59
|
+
|
60
|
+
Simply adding the Gem to your app will configure the client part with useful defaults.
|
61
|
+
|
62
|
+
To enable the background-worker part, you need to be able to automatically start the `backburner` worker.
|
63
|
+
The `backburner` Gem documentation covers this is detail, but they include a God script.
|
64
|
+
|
65
|
+
I've included a set of example Ubuntu Upstart scripts in the `share/upstart` directory, which you will need to update
|
66
|
+
with details like "app_name" and "app_dir" and install to your machine's `/etc/init/` dir.
|
67
|
+
You may also need to make other adjustments to it according to your needs and/or skill.
|
68
|
+
|
69
|
+
Alternatively, you can either start `backburner` in daemon mode with:
|
70
|
+
|
71
|
+
$ cd ${app_dir}
|
72
|
+
$ bundle exec backburner -d -r ${app_config}
|
73
|
+
|
74
|
+
Or wrapped with some-other daemoniser (such as Niet) with:
|
75
|
+
|
76
|
+
$ niet -t backburner -c ${app_dir} -- bundle exec backburner -r ${app_config}
|
77
|
+
|
78
|
+
NB: Replace ${app_dir} with the directory your app is installed to, and ${app_config} with the path to your Squash
|
79
|
+
Repeater config.
|
80
|
+
|
81
|
+
## Configure Squash Repeater
|
82
|
+
|
83
|
+
You need to `require "squash_repeater"` and then use `SquashRepeater.configure` to configure it, e.g:
|
84
|
+
|
85
|
+
```ruby
|
86
|
+
require "squash_repeater"
|
87
|
+
|
88
|
+
SquashRepeater.configure do |c|
|
89
|
+
# The nature of SquashRepeater is that a tiny local queueing system
|
90
|
+
# captures the Squash notification, and retransmits it from a worker.
|
91
|
+
# Therefore, we assume beanstalkd is running locally:
|
92
|
+
|
93
|
+
###
|
94
|
+
# Backburner defaults:
|
95
|
+
#c.backburner.beanstalk_url = "beanstalk://localhost"
|
96
|
+
#c.backburner.tube_namespace = "squash-repeater"
|
97
|
+
#c.backburner.max_job_retries = 10 # retry jobs 10 times
|
98
|
+
#c.backburner.retry_delay = 30 # wait 30 seconds in between retries
|
99
|
+
|
100
|
+
|
101
|
+
###
|
102
|
+
# You can set Squash::Ruby config here, or through their configration method. Either way, they must be set:
|
103
|
+
# @param api_host:
|
104
|
+
#c.squash.api_host = "http://no.where"
|
105
|
+
# @param api_key:
|
106
|
+
#c.squash.api_key = "12345"
|
107
|
+
# @param environment:
|
108
|
+
c.squash.environment = Rails.env if defined? Rails.env
|
109
|
+
# @param disabled:
|
110
|
+
#c.squash.disabled = !c.squash_key
|
111
|
+
|
112
|
+
###
|
113
|
+
# This sets the SquashRepeater and Backburner logging.
|
114
|
+
# There's no easy way to set Squash to use Logger:
|
115
|
+
c.logger = Rails.logger if defined? Rails.logger
|
116
|
+
end
|
117
|
+
```
|
118
|
+
|
119
|
+
As mentioned above, you can configure a few `Squash::Ruby` settings via this config block, or configure it in the Squash
|
120
|
+
way via `Squash::Ruby.configure()`
|
121
|
+
|
122
|
+
## Contributing
|
123
|
+
|
124
|
+
1. Fork it ( https://github.com/powershop/squash_repeater/fork )
|
125
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
126
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
127
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
128
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
|
3
|
+
begin
|
4
|
+
require "rspec/core/rake_task"
|
5
|
+
RSpec::Core::RakeTask.new(:spec) do |task|
|
6
|
+
task.rspec_opts = ["--color"]
|
7
|
+
# task.rspec_opts = ["--color", "--format", "nested"]
|
8
|
+
end
|
9
|
+
rescue LoadError
|
10
|
+
end
|
11
|
+
|
12
|
+
task :default => :spec
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require "rails/generators"
|
2
|
+
|
3
|
+
module SquashRepeater
|
4
|
+
module Generators
|
5
|
+
class InstallGenerator < Rails::Generators::Base
|
6
|
+
source_root File.expand_path("../../templates", __FILE__)
|
7
|
+
desc "Creates SquashRepeater initializer for your application"
|
8
|
+
|
9
|
+
def copy_initializer
|
10
|
+
template "squash_repeater_initializer.rb", "config/initializers/squash_repeater.rb"
|
11
|
+
|
12
|
+
puts "Install complete! Truly Outrageous!"
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require "squash_repeater"
|
2
|
+
|
3
|
+
SquashRepeater.configure do |c|
|
4
|
+
# The nature of SquashRepeater is that a tiny local queueing system
|
5
|
+
# captures the Squash notification, and retransmits it from a worker.
|
6
|
+
# Therefore, we assume beanstalkd is running locally:
|
7
|
+
|
8
|
+
###
|
9
|
+
# Backburner defaults:
|
10
|
+
#c.backburner.beanstalk_url = "beanstalk://localhost"
|
11
|
+
#c.backburner.tube_namespace = "squash-repeater"
|
12
|
+
#c.backburner.max_job_retries = 10 # retry jobs 10 times
|
13
|
+
#c.backburner.retry_delay = 30 # wait 30 seconds in between retries
|
14
|
+
|
15
|
+
###
|
16
|
+
# You can set Squash::Ruby config here, or through their configration method. Either way, they must be set:
|
17
|
+
# @param api_host:
|
18
|
+
#c.squash.api_host = "http://no.where"
|
19
|
+
# @param api_key:
|
20
|
+
#c.squash.api_key = "12345"
|
21
|
+
# @param environment:
|
22
|
+
c.squash.environment = Rails.env if defined? Rails.env
|
23
|
+
# @param disabled:
|
24
|
+
#c.squash.disabled = !c.squash_key
|
25
|
+
|
26
|
+
###
|
27
|
+
# This sets the SquashRepeater and Backburner logging.
|
28
|
+
# There's no easy way to set Squash to use Logger:
|
29
|
+
c.logger = Rails.logger if defined? Rails.logger
|
30
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# Wrap Squash::Ruby's slightly nasty configuration method with
|
2
|
+
# a config-block class
|
3
|
+
#
|
4
|
+
# Squash accepts configuration via a key-value hash passed to the #configure class method
|
5
|
+
# and provides reading that configuration by passing a key the #configuration class method and returning the
|
6
|
+
# value for that key/attribute.
|
7
|
+
# This class attempts to emulate a tradtional Configure-block class (a la Struct / OpenStruct) but calls-into
|
8
|
+
# Squash's configuration methods; this helps make the rest of the SquashRepeater Configure class simpler.
|
9
|
+
# This class is probably more-like OpenStruct in-use, as it doesn't do any checking of "allowed" attributes, which
|
10
|
+
# is more-similar to how Squash treats configuration attr's.
|
11
|
+
# I didn't bother implementing all the methods you might expect, as this should really only be consumed by
|
12
|
+
# SquashRepeater users, at a fairly simplistic level.
|
13
|
+
|
14
|
+
class SquashRepeater::Configuration::Squash
|
15
|
+
def self.configure
|
16
|
+
self.configuration # Initialise
|
17
|
+
yield configuration if block_given?
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.configuration
|
21
|
+
@configuration ||= self.new
|
22
|
+
#return @configuration
|
23
|
+
end
|
24
|
+
|
25
|
+
def method_missing(method_sym, val=nil)
|
26
|
+
fail "Can't extract a meaningful name from the provided method_sym" unless method_sym.to_s =~ /^(.+?)=?$/
|
27
|
+
key = $1.to_sym
|
28
|
+
|
29
|
+
if method_sym.to_s =~ /=$/
|
30
|
+
# Setter
|
31
|
+
Squash::Ruby.configure({ key => val })
|
32
|
+
end
|
33
|
+
# Getter
|
34
|
+
#NB: Always return the value for the key
|
35
|
+
Squash::Ruby.configuration(key)
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require "backburner"
|
2
|
+
require "squash/ruby"
|
3
|
+
|
4
|
+
module SquashRepeater
|
5
|
+
class << self
|
6
|
+
attr_accessor :configuration
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.configure
|
10
|
+
self.configuration ||= Configuration.new # Initialise
|
11
|
+
yield configuration if block_given?
|
12
|
+
end
|
13
|
+
|
14
|
+
class Configuration
|
15
|
+
attr_reader :logger
|
16
|
+
attr_accessor :capture_timeout
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
#NB: You definitely want to think about changing this to something more "substantial"; beanstalkd goes down, you'll lose data.
|
20
|
+
self.logger = Logger.new(STDERR)
|
21
|
+
self.capture_timeout = 2 # seconds
|
22
|
+
|
23
|
+
backburner do |c|
|
24
|
+
# The nature of SquashRepeater is that a tiny local queueing system
|
25
|
+
# captures the Squash notification, and retransmits it from a worker.
|
26
|
+
# Therefore, we assume beanstalkd is running locally:
|
27
|
+
c.beanstalk_url = "beanstalk://localhost"
|
28
|
+
#c.beanstalk_url = "beanstalk://127.0.0.1"
|
29
|
+
c.tube_namespace = "squash-repeater"
|
30
|
+
|
31
|
+
c.max_job_retries = 10 # retry jobs 10 times
|
32
|
+
c.retry_delay = 30 # wait 30 seconds in between retries
|
33
|
+
|
34
|
+
# NB: This relies on forking behaviour!
|
35
|
+
c.default_worker = Backburner::Workers::Forking
|
36
|
+
|
37
|
+
#c.on_error = lambda { |ex| Airbrake.notify(ex) } #FUTURE: Choose a better failure mode:
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def backburner(&p)
|
42
|
+
if block_given?
|
43
|
+
Backburner.configure(&p)
|
44
|
+
else
|
45
|
+
Backburner.configuration
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def squash(&p)
|
50
|
+
if block_given?
|
51
|
+
SquashRepeater::Configuration::Squash.configure(&p)
|
52
|
+
else
|
53
|
+
SquashRepeater::Configuration::Squash.configuration
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# Return an array of all available loggers
|
58
|
+
def loggers
|
59
|
+
[logger, backburner.logger] #FUTURE: Can we somehow get a Squash logger for this?
|
60
|
+
end
|
61
|
+
|
62
|
+
def logger=(value)
|
63
|
+
#FUTURE: Can we somehow also set a Squash logger for this?
|
64
|
+
#NB: Squash doesn't allow you to use a different logger
|
65
|
+
@logger = value
|
66
|
+
#NB: Backburner can be quite chatty. You may prefer to change the default log-level a bit higher because of this.
|
67
|
+
backburner.logger = @logger
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# This class relies on the class hierarchy having been created (above):
|
73
|
+
require "squash_repeater/configure/squash"
|
74
|
+
|
75
|
+
# Set the defaults:
|
76
|
+
SquashRepeater.configure
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require "time"
|
2
|
+
require "timeout"
|
3
|
+
require "backburner"
|
4
|
+
|
5
|
+
module SquashRepeater
|
6
|
+
WORKER_ERRORS = [
|
7
|
+
Beaneater::NotConnected,
|
8
|
+
Beaneater::InvalidTubeName,
|
9
|
+
Beaneater::JobNotReserved,
|
10
|
+
Beaneater::UnexpectedResponse
|
11
|
+
]
|
12
|
+
|
13
|
+
class CaptureTimeoutError < SquashRepeater::Error
|
14
|
+
def to_s
|
15
|
+
original_message = super
|
16
|
+
"Capturing the exception timed-out#{" (#{original_message})" if original_message && !original_message.empty?}"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
WORKER_ERRORS << CaptureTimeoutError
|
20
|
+
|
21
|
+
class << self
|
22
|
+
def transmit_exceptions
|
23
|
+
Backburner.work
|
24
|
+
end
|
25
|
+
alias :work :transmit_exceptions
|
26
|
+
|
27
|
+
# Capture the HTTP data, and store it in the beanstalkd queue for later
|
28
|
+
def capture_exception(url: nil, headers: nil, body: nil, squash_configuration: nil, no_proxy_env: nil)
|
29
|
+
#FUTURE: Required keyword args, for Ruby 2.1+
|
30
|
+
#def capture_exception(url:, headers:, body:, squash_configuration:, no_proxy_env: nil)
|
31
|
+
fail "Missing required keyword arg" unless url && headers && body && squash_configuration
|
32
|
+
|
33
|
+
# If things fail, it's useful to know how long it caused the exception-capture to block the
|
34
|
+
# calling process:
|
35
|
+
start = Time.now
|
36
|
+
|
37
|
+
begin
|
38
|
+
Timeout::timeout(configuration.capture_timeout, CaptureTimeoutError) do
|
39
|
+
#NB: Backburner doesn't seem able to #perform with keyword args:
|
40
|
+
Backburner.enqueue(ExceptionQueue, url, headers, body, squash_configuration, no_proxy_env)
|
41
|
+
end
|
42
|
+
|
43
|
+
rescue *WORKER_ERRORS => e
|
44
|
+
failsafe_handler(
|
45
|
+
e, message: "whilst trying to connect to Beanstalk", time_start: start,
|
46
|
+
args: {
|
47
|
+
url: url,
|
48
|
+
headers: headers,
|
49
|
+
body: body,
|
50
|
+
squash_configuration: squash_configuration,
|
51
|
+
no_proxy_env: no_proxy_env
|
52
|
+
}
|
53
|
+
)
|
54
|
+
raise
|
55
|
+
end
|
56
|
+
end
|
57
|
+
alias :enqueue :capture_exception
|
58
|
+
|
59
|
+
def failsafe_handler(exception, message: nil, time_start: nil, args: {})
|
60
|
+
configuration.logger.error "Failed: #{exception}" + (message && !message.empty? ? ", #{message}." : ".")
|
61
|
+
configuration.logger.error " : #{exception.inspect}"
|
62
|
+
|
63
|
+
configuration.logger.error " (Took #{Time.now - time_start}s to fail)" if time_start
|
64
|
+
configuration.logger.error ["*****"," original_args = #{args.inspect}", "*****"].join("\n")
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
class ExceptionQueue
|
69
|
+
include Backburner::Queue
|
70
|
+
queue "exception"
|
71
|
+
|
72
|
+
# Process one captured Squash notification; i.e. forward it to the Squash
|
73
|
+
# server
|
74
|
+
def self.perform(url, headers, body, squash_configuration, no_proxy_env=nil)
|
75
|
+
#TODO: Change how Squash client deals with failures.
|
76
|
+
# Normally it logs to a special log file, whereas what we really want
|
77
|
+
# is for the job(s) to go back on the queue to be retried.
|
78
|
+
|
79
|
+
# If things fail, it's useful to know how long it caused the exception-capture to block the
|
80
|
+
# calling process:
|
81
|
+
start = Time.now
|
82
|
+
|
83
|
+
#NB: :timeout_protection is a Proc object:
|
84
|
+
squash_configuration = squash_configuration.dup
|
85
|
+
|
86
|
+
#NB: The JSON conversion turns symbol-keys --> strings
|
87
|
+
#NB: Squash::Ruby.configure turns string-keys --> symbols
|
88
|
+
squash_configuration.delete("timeout_protection")
|
89
|
+
|
90
|
+
#NB: This relies on forking behaviour!
|
91
|
+
# We do this, because the queue may be shared, therefore the config may have been different from
|
92
|
+
# each client.
|
93
|
+
Squash::Ruby.configure(squash_configuration)
|
94
|
+
ENV['no_proxy'] = no_proxy_env
|
95
|
+
|
96
|
+
begin
|
97
|
+
# Transmit it to the Squash server:
|
98
|
+
Squash::Ruby.http_transmit__original(url, headers, body)
|
99
|
+
|
100
|
+
rescue SocketError => e
|
101
|
+
SquashRepeater.failsafe_handler(e, message: "whilst trying to connect to Squash", time_start: start)
|
102
|
+
raise
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require "squash/ruby"
|
2
|
+
|
3
|
+
#FUTURE(willjr): Look into using hooks, to try to decouple the two parts of the lib
|
4
|
+
# module Squash::Ruby
|
5
|
+
|
6
|
+
# Monkey-patch Squash::Ruby so that instead of immediately sending Squash
|
7
|
+
# notifications, capture the HTTP data in the local beanstalk/backburner queue
|
8
|
+
module Squash::Ruby
|
9
|
+
class <<self
|
10
|
+
private
|
11
|
+
|
12
|
+
alias :http_transmit__original :http_transmit
|
13
|
+
|
14
|
+
# processing
|
15
|
+
def http_transmit(url, headers, body)
|
16
|
+
SquashRepeater.capture_exception(url: url, headers: headers, body: body, squash_configuration: @configuration, no_proxy_env: ENV["no_proxy"])
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# Configuration methods are in `configure.rb`
|
2
|
+
# Exception capture and re-run are in `exception_queue.rb`
|
3
|
+
|
4
|
+
module SquashRepeater
|
5
|
+
class Error < RuntimeError; end
|
6
|
+
end
|
7
|
+
|
8
|
+
require "squash_repeater/version"
|
9
|
+
require "squash_repeater/configure"
|
10
|
+
require "squash_repeater/exception_queue"
|
11
|
+
require "squash_repeater/squash_ruby"
|
12
|
+
|
13
|
+
# For Rails generators (only if Rails is used):
|
14
|
+
require "generators/squash_repeater/install_generator" if defined? Rails
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# /etc/init/backburner-manager.conf
|
2
|
+
# - Auto-restart a Backburner worker when the time-stamp of the $app_dir changes.
|
3
|
+
# - We assume the changing time-stamp denotes a new deployment of the app.
|
4
|
+
|
5
|
+
# This example config should work with Ubuntu 12.04+. It
|
6
|
+
# allows you to manage multiple Backburner instances with
|
7
|
+
# Upstart.
|
8
|
+
|
9
|
+
env app_name="APP NAME"
|
10
|
+
|
11
|
+
# Change to where your app is installed
|
12
|
+
env app_dir=
|
13
|
+
# Change to where your config is installed; optional
|
14
|
+
env app_config=
|
15
|
+
|
16
|
+
description "Backburner Manager for '$app_name'"
|
17
|
+
|
18
|
+
# This starts upon bootup and stops on shutdown
|
19
|
+
start on runlevel [2345]
|
20
|
+
stop on runlevel [06]
|
21
|
+
|
22
|
+
script
|
23
|
+
start backburner-worker
|
24
|
+
|
25
|
+
while true; do
|
26
|
+
# Gather time-stamp for the app-dir, when started
|
27
|
+
start_app_time=`stat -c %y "$app_dir"`
|
28
|
+
current_app_time=$start_app_time
|
29
|
+
|
30
|
+
logger -t backburner "Time-stamp for '$app_dir' is $start_app_time"
|
31
|
+
|
32
|
+
# Wait for the time-stamp to change:
|
33
|
+
while [ "$current_app_time" = "$start_app_time" ]; do
|
34
|
+
current_app_time=`stat -c %y "$app_dir"`
|
35
|
+
sleep 10
|
36
|
+
done
|
37
|
+
|
38
|
+
logger -t backburner "Time-stamp for '$app_dir' has changed"
|
39
|
+
logger -t backburner "Restarting backburner-worker"
|
40
|
+
|
41
|
+
restart backburner-worker app_name=$app_name app_dir=$app_dir app_config=$app_config
|
42
|
+
done
|
43
|
+
end script
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# /etc/init/backburner-worker.conf - manage a Backburner
|
2
|
+
|
3
|
+
# This example config should work with Ubuntu 12.04+. It
|
4
|
+
# allows you to manage multiple Backburner instances with
|
5
|
+
# Upstart.
|
6
|
+
|
7
|
+
description "Backburner Worker for '$app_name'"
|
8
|
+
|
9
|
+
# No "start on", we want backburner-manager to start us
|
10
|
+
stop on (stopping backburner-manager or runlevel [06])
|
11
|
+
|
12
|
+
# Change to match your deployment user
|
13
|
+
setuid backburner_user
|
14
|
+
setgid backburner_group
|
15
|
+
|
16
|
+
respawn
|
17
|
+
respawn limit 3 30
|
18
|
+
|
19
|
+
script
|
20
|
+
logger -t backburner "Starting Backburner Worker for '$app_name'"
|
21
|
+
|
22
|
+
cd "$app_dir"
|
23
|
+
if [ "$app_config" ]; then
|
24
|
+
bundle exec backburner -d -P /var/run/backburner.pid -r "$app_config"
|
25
|
+
else
|
26
|
+
bundle exec backburner -d -P /var/run/backburner.pid
|
27
|
+
fi
|
28
|
+
end script
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe SquashRepeater::Configuration::Squash do
|
4
|
+
let(:config) { SquashRepeater::Configuration::Squash.configuration }
|
5
|
+
let(:squash_ruby) { class_double("Squash::Ruby").as_stubbed_const(:transfer_nested_constants => true) }
|
6
|
+
|
7
|
+
it do
|
8
|
+
[:set_this, :MosieAlong, :f00b4r].each do |key|
|
9
|
+
expect(squash_ruby).to receive(:configure).with({ key => key.to_s })
|
10
|
+
expect(squash_ruby).to receive(:configuration).with(key)
|
11
|
+
config.send("#{key.to_s}=".to_sym, key.to_s)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
it "when setting and getting a Squash config attr" do
|
16
|
+
[:set_this, :MosieAlong, :f00b4r].each do |key|
|
17
|
+
expect(squash_ruby).to receive(:configuration).with(key).and_return(key.to_s)
|
18
|
+
#expect { config.api_key = "Set this" }.to eq "Set this"
|
19
|
+
expect(config.send(key)).to eq key.to_s
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
require "stringio"
|
4
|
+
require "logger"
|
5
|
+
|
6
|
+
describe SquashRepeater do
|
7
|
+
let(:backburner) { class_double("Backburner").as_stubbed_const(:transfer_nested_constants => true) }
|
8
|
+
let(:backburner_enqueue_args) { ["one", "two", :three, {four: "three"}, :five] }
|
9
|
+
let(:capture_exception_args) { {url: "one", headers: "two", body: :three, squash_configuration: {four: "three"}, no_proxy_env: :five} }
|
10
|
+
|
11
|
+
it do
|
12
|
+
expect(backburner).to receive(:work)
|
13
|
+
SquashRepeater.work
|
14
|
+
end
|
15
|
+
|
16
|
+
it do
|
17
|
+
expect(backburner).to receive(:enqueue).with(SquashRepeater::ExceptionQueue, *backburner_enqueue_args)
|
18
|
+
SquashRepeater.capture_exception(**capture_exception_args)
|
19
|
+
end
|
20
|
+
|
21
|
+
context "within #capture_exception" do
|
22
|
+
let!(:logger_output) { StringIO.new }
|
23
|
+
|
24
|
+
around do |eg|
|
25
|
+
_logger = SquashRepeater.configuration.logger
|
26
|
+
SquashRepeater.configuration.logger = Logger.new logger_output
|
27
|
+
eg.run
|
28
|
+
SquashRepeater.configuration.logger = _logger
|
29
|
+
end
|
30
|
+
|
31
|
+
context "the logger output" do
|
32
|
+
it "for a handled error-type should not be empty" do
|
33
|
+
expect(backburner).to receive(:enqueue).with(SquashRepeater::ExceptionQueue, *backburner_enqueue_args).and_raise(Beaneater::NotConnected)
|
34
|
+
expect { SquashRepeater.capture_exception(**capture_exception_args) }.to raise_error
|
35
|
+
expect(logger_output.string).not_to be_empty
|
36
|
+
end
|
37
|
+
|
38
|
+
it "for a non-handled error-type should be empty" do
|
39
|
+
expect(backburner).to receive(:enqueue).with(SquashRepeater::ExceptionQueue, *backburner_enqueue_args).and_raise(KeyError)
|
40
|
+
expect { SquashRepeater.capture_exception(**capture_exception_args) }.to raise_error
|
41
|
+
expect(logger_output.string).to be_empty
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
|
46
|
+
it do
|
47
|
+
expect(backburner).to receive(:enqueue).with(SquashRepeater::ExceptionQueue, *backburner_enqueue_args).and_raise(Beaneater::NotConnected)
|
48
|
+
expect { SquashRepeater.capture_exception(**capture_exception_args) }.to raise_error
|
49
|
+
end
|
50
|
+
|
51
|
+
#NB: This is technically a medium/large test because of the timeout:
|
52
|
+
it "raises CaptureTimeoutError when worker doesn't return within the timeout" do
|
53
|
+
expect(backburner).to receive(:enqueue).with(SquashRepeater::ExceptionQueue, *backburner_enqueue_args) { sleep SquashRepeater.configuration.capture_timeout + 1 }
|
54
|
+
expect { SquashRepeater.capture_exception(**capture_exception_args) }.to raise_error(SquashRepeater::CaptureTimeoutError)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
describe SquashRepeater::ExceptionQueue do
|
60
|
+
let(:squash_ruby) { class_double("Squash::Ruby").as_stubbed_const(:transfer_nested_constants => true) }
|
61
|
+
|
62
|
+
it do
|
63
|
+
expect(squash_ruby).to receive(:configure).with(
|
64
|
+
config1: "config1", config2: "config2")
|
65
|
+
expect(squash_ruby).to receive(:http_transmit__original).with(
|
66
|
+
"url", "headers", "body"
|
67
|
+
)
|
68
|
+
|
69
|
+
SquashRepeater::ExceptionQueue.perform(
|
70
|
+
"url", "headers", "body", { config1: "config1", config2: "config2" }
|
71
|
+
)
|
72
|
+
end
|
73
|
+
|
74
|
+
it "configuration dup removes 'timeout_protection'" do
|
75
|
+
expect(squash_ruby).to receive(:configure).with(
|
76
|
+
config1: "config1", config2: "config2")
|
77
|
+
expect(squash_ruby).to receive(:http_transmit__original)
|
78
|
+
|
79
|
+
SquashRepeater::ExceptionQueue.perform(
|
80
|
+
"url", "headers", "body",
|
81
|
+
{ config1: "config1", config2: "config2",
|
82
|
+
"timeout_protection" => "should not be here" }
|
83
|
+
)
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe Squash::Ruby do
|
4
|
+
let(:squash_repeater) { class_double("SquashRepeater").as_stubbed_const(:transfer_nested_constants => true) }
|
5
|
+
|
6
|
+
context "with ENV['no_proxy'] unset" do
|
7
|
+
it do
|
8
|
+
#NB: This is a crappy, env-dependent test...
|
9
|
+
expect(ENV["no_proxy"]).to be_nil
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
context "with ENV['no_proxy'] set" do
|
14
|
+
around do |eg|
|
15
|
+
env_no_proxy = ENV["no_proxy"]
|
16
|
+
ENV["no_proxy"] = "TEST no_proxy"
|
17
|
+
eg.run
|
18
|
+
|
19
|
+
ENV["no_proxy"] = env_no_proxy
|
20
|
+
end
|
21
|
+
|
22
|
+
it do
|
23
|
+
expect(squash_repeater).to receive(:capture_exception).with(url: "url", headers: "headers", body: "body", squash_configuration: nil, no_proxy_env: "TEST no_proxy")
|
24
|
+
Squash::Ruby.class_eval { http_transmit("url", "headers", "body") }
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'squash_repeater/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "squash_repeater"
|
8
|
+
spec.version = SquashRepeater::VERSION
|
9
|
+
spec.authors = ["Will Robertson"]
|
10
|
+
spec.email = ["will.robertson@powershop.co.nz"]
|
11
|
+
spec.summary = %q{Squash Repeater for Ruby}
|
12
|
+
spec.description = %q{Use beanstalkd to locally queue and repeat Squash exception capturing.}
|
13
|
+
spec.homepage = ""
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
22
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
23
|
+
|
24
|
+
spec.add_development_dependency "rspec", "~> 3.2"
|
25
|
+
spec.add_development_dependency "simplecov", "~> 0.10"
|
26
|
+
spec.add_development_dependency "pry-byebug", "~> 3.1"
|
27
|
+
|
28
|
+
spec.add_runtime_dependency "squash_ruby", "~> 2.0"
|
29
|
+
spec.add_runtime_dependency "backburner", "~> 0.4"
|
30
|
+
spec.add_runtime_dependency "thor", "~> 0.19"
|
31
|
+
end
|
metadata
ADDED
@@ -0,0 +1,181 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: squash_repeater
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.6
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Will Robertson
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-04-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.7'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.2'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.2'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: simplecov
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0.10'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0.10'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry-byebug
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '3.1'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '3.1'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: squash_ruby
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '2.0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '2.0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: backburner
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0.4'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0.4'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: thor
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0.19'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0.19'
|
125
|
+
description: Use beanstalkd to locally queue and repeat Squash exception capturing.
|
126
|
+
email:
|
127
|
+
- will.robertson@powershop.co.nz
|
128
|
+
executables: []
|
129
|
+
extensions: []
|
130
|
+
extra_rdoc_files: []
|
131
|
+
files:
|
132
|
+
- ".gitignore"
|
133
|
+
- Gemfile
|
134
|
+
- LICENSE.txt
|
135
|
+
- README.md
|
136
|
+
- Rakefile
|
137
|
+
- lib/generators/squash_repeater/install_generator.rb
|
138
|
+
- lib/generators/templates/squash_repeater_initializer.rb
|
139
|
+
- lib/squash_repeater.rb
|
140
|
+
- lib/squash_repeater/configure.rb
|
141
|
+
- lib/squash_repeater/configure/squash.rb
|
142
|
+
- lib/squash_repeater/exception_queue.rb
|
143
|
+
- lib/squash_repeater/squash_ruby.rb
|
144
|
+
- lib/squash_repeater/version.rb
|
145
|
+
- share/upstart/backburner-manager.conf
|
146
|
+
- share/upstart/backburner-worker.conf
|
147
|
+
- spec/spec_helper.rb
|
148
|
+
- spec/squash_repeater_configure_squash.rb
|
149
|
+
- spec/squash_repeater_spec.rb
|
150
|
+
- spec/squash_ruby_spec.rb
|
151
|
+
- squash_repeater-ruby.gemspec
|
152
|
+
homepage: ''
|
153
|
+
licenses:
|
154
|
+
- MIT
|
155
|
+
metadata: {}
|
156
|
+
post_install_message:
|
157
|
+
rdoc_options: []
|
158
|
+
require_paths:
|
159
|
+
- lib
|
160
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
161
|
+
requirements:
|
162
|
+
- - ">="
|
163
|
+
- !ruby/object:Gem::Version
|
164
|
+
version: '0'
|
165
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
166
|
+
requirements:
|
167
|
+
- - ">="
|
168
|
+
- !ruby/object:Gem::Version
|
169
|
+
version: '0'
|
170
|
+
requirements: []
|
171
|
+
rubyforge_project:
|
172
|
+
rubygems_version: 2.4.5
|
173
|
+
signing_key:
|
174
|
+
specification_version: 4
|
175
|
+
summary: Squash Repeater for Ruby
|
176
|
+
test_files:
|
177
|
+
- spec/spec_helper.rb
|
178
|
+
- spec/squash_repeater_configure_squash.rb
|
179
|
+
- spec/squash_repeater_spec.rb
|
180
|
+
- spec/squash_ruby_spec.rb
|
181
|
+
has_rdoc:
|