squash_repeater 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +128 -0
- data/Rakefile +12 -0
- data/lib/generators/squash_repeater/install_generator.rb +16 -0
- data/lib/generators/templates/squash_repeater_initializer.rb +30 -0
- data/lib/squash_repeater/configure/squash.rb +37 -0
- data/lib/squash_repeater/configure.rb +76 -0
- data/lib/squash_repeater/exception_queue.rb +106 -0
- data/lib/squash_repeater/squash_ruby.rb +19 -0
- data/lib/squash_repeater/version.rb +3 -0
- data/lib/squash_repeater.rb +14 -0
- data/share/upstart/backburner-manager.conf +43 -0
- data/share/upstart/backburner-worker.conf +28 -0
- data/spec/spec_helper.rb +11 -0
- data/spec/squash_repeater_configure_squash.rb +22 -0
- data/spec/squash_repeater_spec.rb +85 -0
- data/spec/squash_ruby_spec.rb +27 -0
- data/squash_repeater-ruby.gemspec +31 -0
- metadata +181 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 219ecac4dc1a24819c44deee687c56d0cadbe2b2
|
4
|
+
data.tar.gz: cf636f9dd0f10862986aa4f09041fd16705f6f19
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c93756e6d025867532c462cc38fb74c46584ffca9ea47cba2f113697e2ab39d520fc0729dc503990fa94affe0c41b1b90cab4099509f26c487150b74ab0811df
|
7
|
+
data.tar.gz: cef8c48c93aea8e1e1136eb93854aff7a790e8717b2bb5ad508828be3720bb749d3e42a8937bd066724a799bf6181485c4ed57eeebfbf0262fe8866bf61c8ef5
|
data/.gitignore
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
# Specify your gem's dependencies in squash_repeater-ruby.gemspec
|
4
|
+
gemspec
|
5
|
+
|
6
|
+
# Latest beaneater has an important fix (see https://github.com/powershop/squash_repeater/issues/2):
|
7
|
+
# You may need to add this to your including library until a new gem of beaneater is released:
|
8
|
+
#gem 'beaneater', :github => 'beanstalkd/beaneater'
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2015 Powershop
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,128 @@
|
|
1
|
+
# SquashRepeater
|
2
|
+
|
3
|
+
One difficulty with Squash is that whenever any exception occurs, it contacts the
|
4
|
+
Squash server to upload details of that failure, which means the code blocks until
|
5
|
+
contact is made or the connection times-out.
|
6
|
+
|
7
|
+
In many cases, an exception is a failure, and you probably don't want to be nice
|
8
|
+
about it, but for user-facing app's, you're degrading the user-experience even
|
9
|
+
further. A quick-response "failure" message to the user is still many times better
|
10
|
+
than waiting many seconds (or more).
|
11
|
+
|
12
|
+
On top of that, almost all exceptions are valuable, in that they have captured a
|
13
|
+
failure-mode that you weren't previously aware of. If the server is down, then the
|
14
|
+
chances are that data is lost.
|
15
|
+
|
16
|
+
Squash Repeater uses a low-overhead queueing service to capture any exceptions and
|
17
|
+
return control as quickly as possible, after which an independent "worker" process
|
18
|
+
will attempt to send the queued exception reports to the Squash server.
|
19
|
+
This should mean:
|
20
|
+
- the local Squash client code can return far more quickly
|
21
|
+
- any exception reports that fail to be accepted by the Squash server aren't lost
|
22
|
+
(or dropped from the queue)
|
23
|
+
- we can "distribute" part of the Squash service throughout all the client servers
|
24
|
+
|
25
|
+
It doesn't solve things like
|
26
|
+
- multi-homed / distributed Squash server
|
27
|
+
|
28
|
+
## Installation
|
29
|
+
|
30
|
+
Add this line to your application's Gemfile:
|
31
|
+
|
32
|
+
```ruby
|
33
|
+
gem 'squash_repeater', github: 'powershop/squash_repeater'
|
34
|
+
```
|
35
|
+
|
36
|
+
And then execute:
|
37
|
+
|
38
|
+
$ bundle
|
39
|
+
|
40
|
+
Or install it yourself as:
|
41
|
+
|
42
|
+
$ gem install squash_repeater
|
43
|
+
|
44
|
+
If you're using Rails, you can install an initialiser template with:
|
45
|
+
|
46
|
+
$ bundle exec rails generate squash_repeater:install
|
47
|
+
|
48
|
+
## Install `beanstalkd`
|
49
|
+
|
50
|
+
- Configure it to listen on `localhost` network
|
51
|
+
- Configure it to auto-start on system boot
|
52
|
+
- Give it a persistent store for queue data (needed in-order to not lose data)
|
53
|
+
|
54
|
+
## Configure `backburner`
|
55
|
+
|
56
|
+
`backburner` is the Gem used to interact with the `beanstalk` queue.
|
57
|
+
It works in two parts: client libraries that put data on the queue, and background-worker jobs that process the data on
|
58
|
+
queue.
|
59
|
+
|
60
|
+
Simply adding the Gem to your app will configure the client part with useful defaults.
|
61
|
+
|
62
|
+
To enable the background-worker part, you need to be able to automatically start the `backburner` worker.
|
63
|
+
The `backburner` Gem documentation covers this is detail, but they include a God script.
|
64
|
+
|
65
|
+
I've included a set of example Ubuntu Upstart scripts in the `share/upstart` directory, which you will need to update
|
66
|
+
with details like "app_name" and "app_dir" and install to your machine's `/etc/init/` dir.
|
67
|
+
You may also need to make other adjustments to it according to your needs and/or skill.
|
68
|
+
|
69
|
+
Alternatively, you can either start `backburner` in daemon mode with:
|
70
|
+
|
71
|
+
$ cd ${app_dir}
|
72
|
+
$ bundle exec backburner -d -r ${app_config}
|
73
|
+
|
74
|
+
Or wrapped with some-other daemoniser (such as Niet) with:
|
75
|
+
|
76
|
+
$ niet -t backburner -c ${app_dir} -- bundle exec backburner -r ${app_config}
|
77
|
+
|
78
|
+
NB: Replace ${app_dir} with the directory your app is installed to, and ${app_config} with the path to your Squash
|
79
|
+
Repeater config.
|
80
|
+
|
81
|
+
## Configure Squash Repeater
|
82
|
+
|
83
|
+
You need to `require "squash_repeater"` and then use `SquashRepeater.configure` to configure it, e.g:
|
84
|
+
|
85
|
+
```ruby
|
86
|
+
require "squash_repeater"
|
87
|
+
|
88
|
+
SquashRepeater.configure do |c|
|
89
|
+
# The nature of SquashRepeater is that a tiny local queueing system
|
90
|
+
# captures the Squash notification, and retransmits it from a worker.
|
91
|
+
# Therefore, we assume beanstalkd is running locally:
|
92
|
+
|
93
|
+
###
|
94
|
+
# Backburner defaults:
|
95
|
+
#c.backburner.beanstalk_url = "beanstalk://localhost"
|
96
|
+
#c.backburner.tube_namespace = "squash-repeater"
|
97
|
+
#c.backburner.max_job_retries = 10 # retry jobs 10 times
|
98
|
+
#c.backburner.retry_delay = 30 # wait 30 seconds in between retries
|
99
|
+
|
100
|
+
|
101
|
+
###
|
102
|
+
# You can set Squash::Ruby config here, or through their configration method. Either way, they must be set:
|
103
|
+
# @param api_host:
|
104
|
+
#c.squash.api_host = "http://no.where"
|
105
|
+
# @param api_key:
|
106
|
+
#c.squash.api_key = "12345"
|
107
|
+
# @param environment:
|
108
|
+
c.squash.environment = Rails.env if defined? Rails.env
|
109
|
+
# @param disabled:
|
110
|
+
#c.squash.disabled = !c.squash_key
|
111
|
+
|
112
|
+
###
|
113
|
+
# This sets the SquashRepeater and Backburner logging.
|
114
|
+
# There's no easy way to set Squash to use Logger:
|
115
|
+
c.logger = Rails.logger if defined? Rails.logger
|
116
|
+
end
|
117
|
+
```
|
118
|
+
|
119
|
+
As mentioned above, you can configure a few `Squash::Ruby` settings via this config block, or configure it in the Squash
|
120
|
+
way via `Squash::Ruby.configure()`
|
121
|
+
|
122
|
+
## Contributing
|
123
|
+
|
124
|
+
1. Fork it ( https://github.com/powershop/squash_repeater/fork )
|
125
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
126
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
127
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
128
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
|
3
|
+
begin
|
4
|
+
require "rspec/core/rake_task"
|
5
|
+
RSpec::Core::RakeTask.new(:spec) do |task|
|
6
|
+
task.rspec_opts = ["--color"]
|
7
|
+
# task.rspec_opts = ["--color", "--format", "nested"]
|
8
|
+
end
|
9
|
+
rescue LoadError
|
10
|
+
end
|
11
|
+
|
12
|
+
task :default => :spec
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require "rails/generators"
|
2
|
+
|
3
|
+
module SquashRepeater
|
4
|
+
module Generators
|
5
|
+
class InstallGenerator < Rails::Generators::Base
|
6
|
+
source_root File.expand_path("../../templates", __FILE__)
|
7
|
+
desc "Creates SquashRepeater initializer for your application"
|
8
|
+
|
9
|
+
def copy_initializer
|
10
|
+
template "squash_repeater_initializer.rb", "config/initializers/squash_repeater.rb"
|
11
|
+
|
12
|
+
puts "Install complete! Truly Outrageous!"
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require "squash_repeater"
|
2
|
+
|
3
|
+
SquashRepeater.configure do |c|
|
4
|
+
# The nature of SquashRepeater is that a tiny local queueing system
|
5
|
+
# captures the Squash notification, and retransmits it from a worker.
|
6
|
+
# Therefore, we assume beanstalkd is running locally:
|
7
|
+
|
8
|
+
###
|
9
|
+
# Backburner defaults:
|
10
|
+
#c.backburner.beanstalk_url = "beanstalk://localhost"
|
11
|
+
#c.backburner.tube_namespace = "squash-repeater"
|
12
|
+
#c.backburner.max_job_retries = 10 # retry jobs 10 times
|
13
|
+
#c.backburner.retry_delay = 30 # wait 30 seconds in between retries
|
14
|
+
|
15
|
+
###
|
16
|
+
# You can set Squash::Ruby config here, or through their configration method. Either way, they must be set:
|
17
|
+
# @param api_host:
|
18
|
+
#c.squash.api_host = "http://no.where"
|
19
|
+
# @param api_key:
|
20
|
+
#c.squash.api_key = "12345"
|
21
|
+
# @param environment:
|
22
|
+
c.squash.environment = Rails.env if defined? Rails.env
|
23
|
+
# @param disabled:
|
24
|
+
#c.squash.disabled = !c.squash_key
|
25
|
+
|
26
|
+
###
|
27
|
+
# This sets the SquashRepeater and Backburner logging.
|
28
|
+
# There's no easy way to set Squash to use Logger:
|
29
|
+
c.logger = Rails.logger if defined? Rails.logger
|
30
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# Wrap Squash::Ruby's slightly nasty configuration method with
|
2
|
+
# a config-block class
|
3
|
+
#
|
4
|
+
# Squash accepts configuration via a key-value hash passed to the #configure class method
|
5
|
+
# and provides reading that configuration by passing a key the #configuration class method and returning the
|
6
|
+
# value for that key/attribute.
|
7
|
+
# This class attempts to emulate a tradtional Configure-block class (a la Struct / OpenStruct) but calls-into
|
8
|
+
# Squash's configuration methods; this helps make the rest of the SquashRepeater Configure class simpler.
|
9
|
+
# This class is probably more-like OpenStruct in-use, as it doesn't do any checking of "allowed" attributes, which
|
10
|
+
# is more-similar to how Squash treats configuration attr's.
|
11
|
+
# I didn't bother implementing all the methods you might expect, as this should really only be consumed by
|
12
|
+
# SquashRepeater users, at a fairly simplistic level.
|
13
|
+
|
14
|
+
class SquashRepeater::Configuration::Squash
|
15
|
+
def self.configure
|
16
|
+
self.configuration # Initialise
|
17
|
+
yield configuration if block_given?
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.configuration
|
21
|
+
@configuration ||= self.new
|
22
|
+
#return @configuration
|
23
|
+
end
|
24
|
+
|
25
|
+
def method_missing(method_sym, val=nil)
|
26
|
+
fail "Can't extract a meaningful name from the provided method_sym" unless method_sym.to_s =~ /^(.+?)=?$/
|
27
|
+
key = $1.to_sym
|
28
|
+
|
29
|
+
if method_sym.to_s =~ /=$/
|
30
|
+
# Setter
|
31
|
+
Squash::Ruby.configure({ key => val })
|
32
|
+
end
|
33
|
+
# Getter
|
34
|
+
#NB: Always return the value for the key
|
35
|
+
Squash::Ruby.configuration(key)
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require "backburner"
|
2
|
+
require "squash/ruby"
|
3
|
+
|
4
|
+
module SquashRepeater
|
5
|
+
class << self
|
6
|
+
attr_accessor :configuration
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.configure
|
10
|
+
self.configuration ||= Configuration.new # Initialise
|
11
|
+
yield configuration if block_given?
|
12
|
+
end
|
13
|
+
|
14
|
+
class Configuration
|
15
|
+
attr_reader :logger
|
16
|
+
attr_accessor :capture_timeout
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
#NB: You definitely want to think about changing this to something more "substantial"; beanstalkd goes down, you'll lose data.
|
20
|
+
self.logger = Logger.new(STDERR)
|
21
|
+
self.capture_timeout = 2 # seconds
|
22
|
+
|
23
|
+
backburner do |c|
|
24
|
+
# The nature of SquashRepeater is that a tiny local queueing system
|
25
|
+
# captures the Squash notification, and retransmits it from a worker.
|
26
|
+
# Therefore, we assume beanstalkd is running locally:
|
27
|
+
c.beanstalk_url = "beanstalk://localhost"
|
28
|
+
#c.beanstalk_url = "beanstalk://127.0.0.1"
|
29
|
+
c.tube_namespace = "squash-repeater"
|
30
|
+
|
31
|
+
c.max_job_retries = 10 # retry jobs 10 times
|
32
|
+
c.retry_delay = 30 # wait 30 seconds in between retries
|
33
|
+
|
34
|
+
# NB: This relies on forking behaviour!
|
35
|
+
c.default_worker = Backburner::Workers::Forking
|
36
|
+
|
37
|
+
#c.on_error = lambda { |ex| Airbrake.notify(ex) } #FUTURE: Choose a better failure mode:
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def backburner(&p)
|
42
|
+
if block_given?
|
43
|
+
Backburner.configure(&p)
|
44
|
+
else
|
45
|
+
Backburner.configuration
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def squash(&p)
|
50
|
+
if block_given?
|
51
|
+
SquashRepeater::Configuration::Squash.configure(&p)
|
52
|
+
else
|
53
|
+
SquashRepeater::Configuration::Squash.configuration
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# Return an array of all available loggers
|
58
|
+
def loggers
|
59
|
+
[logger, backburner.logger] #FUTURE: Can we somehow get a Squash logger for this?
|
60
|
+
end
|
61
|
+
|
62
|
+
def logger=(value)
|
63
|
+
#FUTURE: Can we somehow also set a Squash logger for this?
|
64
|
+
#NB: Squash doesn't allow you to use a different logger
|
65
|
+
@logger = value
|
66
|
+
#NB: Backburner can be quite chatty. You may prefer to change the default log-level a bit higher because of this.
|
67
|
+
backburner.logger = @logger
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# This class relies on the class hierarchy having been created (above):
|
73
|
+
require "squash_repeater/configure/squash"
|
74
|
+
|
75
|
+
# Set the defaults:
|
76
|
+
SquashRepeater.configure
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require "time"
|
2
|
+
require "timeout"
|
3
|
+
require "backburner"
|
4
|
+
|
5
|
+
module SquashRepeater
|
6
|
+
WORKER_ERRORS = [
|
7
|
+
Beaneater::NotConnected,
|
8
|
+
Beaneater::InvalidTubeName,
|
9
|
+
Beaneater::JobNotReserved,
|
10
|
+
Beaneater::UnexpectedResponse
|
11
|
+
]
|
12
|
+
|
13
|
+
class CaptureTimeoutError < SquashRepeater::Error
|
14
|
+
def to_s
|
15
|
+
original_message = super
|
16
|
+
"Capturing the exception timed-out#{" (#{original_message})" if original_message && !original_message.empty?}"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
WORKER_ERRORS << CaptureTimeoutError
|
20
|
+
|
21
|
+
class << self
|
22
|
+
def transmit_exceptions
|
23
|
+
Backburner.work
|
24
|
+
end
|
25
|
+
alias :work :transmit_exceptions
|
26
|
+
|
27
|
+
# Capture the HTTP data, and store it in the beanstalkd queue for later
|
28
|
+
def capture_exception(url: nil, headers: nil, body: nil, squash_configuration: nil, no_proxy_env: nil)
|
29
|
+
#FUTURE: Required keyword args, for Ruby 2.1+
|
30
|
+
#def capture_exception(url:, headers:, body:, squash_configuration:, no_proxy_env: nil)
|
31
|
+
fail "Missing required keyword arg" unless url && headers && body && squash_configuration
|
32
|
+
|
33
|
+
# If things fail, it's useful to know how long it caused the exception-capture to block the
|
34
|
+
# calling process:
|
35
|
+
start = Time.now
|
36
|
+
|
37
|
+
begin
|
38
|
+
Timeout::timeout(configuration.capture_timeout, CaptureTimeoutError) do
|
39
|
+
#NB: Backburner doesn't seem able to #perform with keyword args:
|
40
|
+
Backburner.enqueue(ExceptionQueue, url, headers, body, squash_configuration, no_proxy_env)
|
41
|
+
end
|
42
|
+
|
43
|
+
rescue *WORKER_ERRORS => e
|
44
|
+
failsafe_handler(
|
45
|
+
e, message: "whilst trying to connect to Beanstalk", time_start: start,
|
46
|
+
args: {
|
47
|
+
url: url,
|
48
|
+
headers: headers,
|
49
|
+
body: body,
|
50
|
+
squash_configuration: squash_configuration,
|
51
|
+
no_proxy_env: no_proxy_env
|
52
|
+
}
|
53
|
+
)
|
54
|
+
raise
|
55
|
+
end
|
56
|
+
end
|
57
|
+
alias :enqueue :capture_exception
|
58
|
+
|
59
|
+
def failsafe_handler(exception, message: nil, time_start: nil, args: {})
|
60
|
+
configuration.logger.error "Failed: #{exception}" + (message && !message.empty? ? ", #{message}." : ".")
|
61
|
+
configuration.logger.error " : #{exception.inspect}"
|
62
|
+
|
63
|
+
configuration.logger.error " (Took #{Time.now - time_start}s to fail)" if time_start
|
64
|
+
configuration.logger.error ["*****"," original_args = #{args.inspect}", "*****"].join("\n")
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
class ExceptionQueue
|
69
|
+
include Backburner::Queue
|
70
|
+
queue "exception"
|
71
|
+
|
72
|
+
# Process one captured Squash notification; i.e. forward it to the Squash
|
73
|
+
# server
|
74
|
+
def self.perform(url, headers, body, squash_configuration, no_proxy_env=nil)
|
75
|
+
#TODO: Change how Squash client deals with failures.
|
76
|
+
# Normally it logs to a special log file, whereas what we really want
|
77
|
+
# is for the job(s) to go back on the queue to be retried.
|
78
|
+
|
79
|
+
# If things fail, it's useful to know how long it caused the exception-capture to block the
|
80
|
+
# calling process:
|
81
|
+
start = Time.now
|
82
|
+
|
83
|
+
#NB: :timeout_protection is a Proc object:
|
84
|
+
squash_configuration = squash_configuration.dup
|
85
|
+
|
86
|
+
#NB: The JSON conversion turns symbol-keys --> strings
|
87
|
+
#NB: Squash::Ruby.configure turns string-keys --> symbols
|
88
|
+
squash_configuration.delete("timeout_protection")
|
89
|
+
|
90
|
+
#NB: This relies on forking behaviour!
|
91
|
+
# We do this, because the queue may be shared, therefore the config may have been different from
|
92
|
+
# each client.
|
93
|
+
Squash::Ruby.configure(squash_configuration)
|
94
|
+
ENV['no_proxy'] = no_proxy_env
|
95
|
+
|
96
|
+
begin
|
97
|
+
# Transmit it to the Squash server:
|
98
|
+
Squash::Ruby.http_transmit__original(url, headers, body)
|
99
|
+
|
100
|
+
rescue SocketError => e
|
101
|
+
SquashRepeater.failsafe_handler(e, message: "whilst trying to connect to Squash", time_start: start)
|
102
|
+
raise
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require "squash/ruby"
|
2
|
+
|
3
|
+
#FUTURE(willjr): Look into using hooks, to try to decouple the two parts of the lib
|
4
|
+
# module Squash::Ruby
|
5
|
+
|
6
|
+
# Monkey-patch Squash::Ruby so that instead of immediately sending Squash
|
7
|
+
# notifications, capture the HTTP data in the local beanstalk/backburner queue
|
8
|
+
module Squash::Ruby
|
9
|
+
class <<self
|
10
|
+
private
|
11
|
+
|
12
|
+
alias :http_transmit__original :http_transmit
|
13
|
+
|
14
|
+
# processing
|
15
|
+
def http_transmit(url, headers, body)
|
16
|
+
SquashRepeater.capture_exception(url: url, headers: headers, body: body, squash_configuration: @configuration, no_proxy_env: ENV["no_proxy"])
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# Configuration methods are in `configure.rb`
|
2
|
+
# Exception capture and re-run are in `exception_queue.rb`
|
3
|
+
|
4
|
+
module SquashRepeater
|
5
|
+
class Error < RuntimeError; end
|
6
|
+
end
|
7
|
+
|
8
|
+
require "squash_repeater/version"
|
9
|
+
require "squash_repeater/configure"
|
10
|
+
require "squash_repeater/exception_queue"
|
11
|
+
require "squash_repeater/squash_ruby"
|
12
|
+
|
13
|
+
# For Rails generators (only if Rails is used):
|
14
|
+
require "generators/squash_repeater/install_generator" if defined? Rails
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# /etc/init/backburner-manager.conf
|
2
|
+
# - Auto-restart a Backburner worker when the time-stamp of the $app_dir changes.
|
3
|
+
# - We assume the changing time-stamp denotes a new deployment of the app.
|
4
|
+
|
5
|
+
# This example config should work with Ubuntu 12.04+. It
|
6
|
+
# allows you to manage multiple Backburner instances with
|
7
|
+
# Upstart.
|
8
|
+
|
9
|
+
env app_name="APP NAME"
|
10
|
+
|
11
|
+
# Change to where your app is installed
|
12
|
+
env app_dir=
|
13
|
+
# Change to where your config is installed; optional
|
14
|
+
env app_config=
|
15
|
+
|
16
|
+
description "Backburner Manager for '$app_name'"
|
17
|
+
|
18
|
+
# This starts upon bootup and stops on shutdown
|
19
|
+
start on runlevel [2345]
|
20
|
+
stop on runlevel [06]
|
21
|
+
|
22
|
+
script
|
23
|
+
start backburner-worker
|
24
|
+
|
25
|
+
while true; do
|
26
|
+
# Gather time-stamp for the app-dir, when started
|
27
|
+
start_app_time=`stat -c %y "$app_dir"`
|
28
|
+
current_app_time=$start_app_time
|
29
|
+
|
30
|
+
logger -t backburner "Time-stamp for '$app_dir' is $start_app_time"
|
31
|
+
|
32
|
+
# Wait for the time-stamp to change:
|
33
|
+
while [ "$current_app_time" = "$start_app_time" ]; do
|
34
|
+
current_app_time=`stat -c %y "$app_dir"`
|
35
|
+
sleep 10
|
36
|
+
done
|
37
|
+
|
38
|
+
logger -t backburner "Time-stamp for '$app_dir' has changed"
|
39
|
+
logger -t backburner "Restarting backburner-worker"
|
40
|
+
|
41
|
+
restart backburner-worker app_name=$app_name app_dir=$app_dir app_config=$app_config
|
42
|
+
done
|
43
|
+
end script
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# /etc/init/backburner-worker.conf - manage a Backburner
|
2
|
+
|
3
|
+
# This example config should work with Ubuntu 12.04+. It
|
4
|
+
# allows you to manage multiple Backburner instances with
|
5
|
+
# Upstart.
|
6
|
+
|
7
|
+
description "Backburner Worker for '$app_name'"
|
8
|
+
|
9
|
+
# No "start on", we want backburner-manager to start us
|
10
|
+
stop on (stopping backburner-manager or runlevel [06])
|
11
|
+
|
12
|
+
# Change to match your deployment user
|
13
|
+
setuid backburner_user
|
14
|
+
setgid backburner_group
|
15
|
+
|
16
|
+
respawn
|
17
|
+
respawn limit 3 30
|
18
|
+
|
19
|
+
script
|
20
|
+
logger -t backburner "Starting Backburner Worker for '$app_name'"
|
21
|
+
|
22
|
+
cd "$app_dir"
|
23
|
+
if [ "$app_config" ]; then
|
24
|
+
bundle exec backburner -d -P /var/run/backburner.pid -r "$app_config"
|
25
|
+
else
|
26
|
+
bundle exec backburner -d -P /var/run/backburner.pid
|
27
|
+
fi
|
28
|
+
end script
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe SquashRepeater::Configuration::Squash do
|
4
|
+
let(:config) { SquashRepeater::Configuration::Squash.configuration }
|
5
|
+
let(:squash_ruby) { class_double("Squash::Ruby").as_stubbed_const(:transfer_nested_constants => true) }
|
6
|
+
|
7
|
+
it do
|
8
|
+
[:set_this, :MosieAlong, :f00b4r].each do |key|
|
9
|
+
expect(squash_ruby).to receive(:configure).with({ key => key.to_s })
|
10
|
+
expect(squash_ruby).to receive(:configuration).with(key)
|
11
|
+
config.send("#{key.to_s}=".to_sym, key.to_s)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
it "when setting and getting a Squash config attr" do
|
16
|
+
[:set_this, :MosieAlong, :f00b4r].each do |key|
|
17
|
+
expect(squash_ruby).to receive(:configuration).with(key).and_return(key.to_s)
|
18
|
+
#expect { config.api_key = "Set this" }.to eq "Set this"
|
19
|
+
expect(config.send(key)).to eq key.to_s
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
require "stringio"
|
4
|
+
require "logger"
|
5
|
+
|
6
|
+
describe SquashRepeater do
|
7
|
+
let(:backburner) { class_double("Backburner").as_stubbed_const(:transfer_nested_constants => true) }
|
8
|
+
let(:backburner_enqueue_args) { ["one", "two", :three, {four: "three"}, :five] }
|
9
|
+
let(:capture_exception_args) { {url: "one", headers: "two", body: :three, squash_configuration: {four: "three"}, no_proxy_env: :five} }
|
10
|
+
|
11
|
+
it do
|
12
|
+
expect(backburner).to receive(:work)
|
13
|
+
SquashRepeater.work
|
14
|
+
end
|
15
|
+
|
16
|
+
it do
|
17
|
+
expect(backburner).to receive(:enqueue).with(SquashRepeater::ExceptionQueue, *backburner_enqueue_args)
|
18
|
+
SquashRepeater.capture_exception(**capture_exception_args)
|
19
|
+
end
|
20
|
+
|
21
|
+
context "within #capture_exception" do
|
22
|
+
let!(:logger_output) { StringIO.new }
|
23
|
+
|
24
|
+
around do |eg|
|
25
|
+
_logger = SquashRepeater.configuration.logger
|
26
|
+
SquashRepeater.configuration.logger = Logger.new logger_output
|
27
|
+
eg.run
|
28
|
+
SquashRepeater.configuration.logger = _logger
|
29
|
+
end
|
30
|
+
|
31
|
+
context "the logger output" do
|
32
|
+
it "for a handled error-type should not be empty" do
|
33
|
+
expect(backburner).to receive(:enqueue).with(SquashRepeater::ExceptionQueue, *backburner_enqueue_args).and_raise(Beaneater::NotConnected)
|
34
|
+
expect { SquashRepeater.capture_exception(**capture_exception_args) }.to raise_error
|
35
|
+
expect(logger_output.string).not_to be_empty
|
36
|
+
end
|
37
|
+
|
38
|
+
it "for a non-handled error-type should be empty" do
|
39
|
+
expect(backburner).to receive(:enqueue).with(SquashRepeater::ExceptionQueue, *backburner_enqueue_args).and_raise(KeyError)
|
40
|
+
expect { SquashRepeater.capture_exception(**capture_exception_args) }.to raise_error
|
41
|
+
expect(logger_output.string).to be_empty
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
|
46
|
+
it do
|
47
|
+
expect(backburner).to receive(:enqueue).with(SquashRepeater::ExceptionQueue, *backburner_enqueue_args).and_raise(Beaneater::NotConnected)
|
48
|
+
expect { SquashRepeater.capture_exception(**capture_exception_args) }.to raise_error
|
49
|
+
end
|
50
|
+
|
51
|
+
#NB: This is technically a medium/large test because of the timeout:
|
52
|
+
it "raises CaptureTimeoutError when worker doesn't return within the timeout" do
|
53
|
+
expect(backburner).to receive(:enqueue).with(SquashRepeater::ExceptionQueue, *backburner_enqueue_args) { sleep SquashRepeater.configuration.capture_timeout + 1 }
|
54
|
+
expect { SquashRepeater.capture_exception(**capture_exception_args) }.to raise_error(SquashRepeater::CaptureTimeoutError)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
describe SquashRepeater::ExceptionQueue do
|
60
|
+
let(:squash_ruby) { class_double("Squash::Ruby").as_stubbed_const(:transfer_nested_constants => true) }
|
61
|
+
|
62
|
+
it do
|
63
|
+
expect(squash_ruby).to receive(:configure).with(
|
64
|
+
config1: "config1", config2: "config2")
|
65
|
+
expect(squash_ruby).to receive(:http_transmit__original).with(
|
66
|
+
"url", "headers", "body"
|
67
|
+
)
|
68
|
+
|
69
|
+
SquashRepeater::ExceptionQueue.perform(
|
70
|
+
"url", "headers", "body", { config1: "config1", config2: "config2" }
|
71
|
+
)
|
72
|
+
end
|
73
|
+
|
74
|
+
it "configuration dup removes 'timeout_protection'" do
|
75
|
+
expect(squash_ruby).to receive(:configure).with(
|
76
|
+
config1: "config1", config2: "config2")
|
77
|
+
expect(squash_ruby).to receive(:http_transmit__original)
|
78
|
+
|
79
|
+
SquashRepeater::ExceptionQueue.perform(
|
80
|
+
"url", "headers", "body",
|
81
|
+
{ config1: "config1", config2: "config2",
|
82
|
+
"timeout_protection" => "should not be here" }
|
83
|
+
)
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe Squash::Ruby do
|
4
|
+
let(:squash_repeater) { class_double("SquashRepeater").as_stubbed_const(:transfer_nested_constants => true) }
|
5
|
+
|
6
|
+
context "with ENV['no_proxy'] unset" do
|
7
|
+
it do
|
8
|
+
#NB: This is a crappy, env-dependent test...
|
9
|
+
expect(ENV["no_proxy"]).to be_nil
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
context "with ENV['no_proxy'] set" do
|
14
|
+
around do |eg|
|
15
|
+
env_no_proxy = ENV["no_proxy"]
|
16
|
+
ENV["no_proxy"] = "TEST no_proxy"
|
17
|
+
eg.run
|
18
|
+
|
19
|
+
ENV["no_proxy"] = env_no_proxy
|
20
|
+
end
|
21
|
+
|
22
|
+
it do
|
23
|
+
expect(squash_repeater).to receive(:capture_exception).with(url: "url", headers: "headers", body: "body", squash_configuration: nil, no_proxy_env: "TEST no_proxy")
|
24
|
+
Squash::Ruby.class_eval { http_transmit("url", "headers", "body") }
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'squash_repeater/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "squash_repeater"
|
8
|
+
spec.version = SquashRepeater::VERSION
|
9
|
+
spec.authors = ["Will Robertson"]
|
10
|
+
spec.email = ["will.robertson@powershop.co.nz"]
|
11
|
+
spec.summary = %q{Squash Repeater for Ruby}
|
12
|
+
spec.description = %q{Use beanstalkd to locally queue and repeat Squash exception capturing.}
|
13
|
+
spec.homepage = ""
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
22
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
23
|
+
|
24
|
+
spec.add_development_dependency "rspec", "~> 3.2"
|
25
|
+
spec.add_development_dependency "simplecov", "~> 0.10"
|
26
|
+
spec.add_development_dependency "pry-byebug", "~> 3.1"
|
27
|
+
|
28
|
+
spec.add_runtime_dependency "squash_ruby", "~> 2.0"
|
29
|
+
spec.add_runtime_dependency "backburner", "~> 0.4"
|
30
|
+
spec.add_runtime_dependency "thor", "~> 0.19"
|
31
|
+
end
|
metadata
ADDED
@@ -0,0 +1,181 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: squash_repeater
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.6
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Will Robertson
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-04-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.7'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.2'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.2'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: simplecov
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0.10'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0.10'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry-byebug
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '3.1'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '3.1'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: squash_ruby
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '2.0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '2.0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: backburner
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0.4'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0.4'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: thor
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0.19'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0.19'
|
125
|
+
description: Use beanstalkd to locally queue and repeat Squash exception capturing.
|
126
|
+
email:
|
127
|
+
- will.robertson@powershop.co.nz
|
128
|
+
executables: []
|
129
|
+
extensions: []
|
130
|
+
extra_rdoc_files: []
|
131
|
+
files:
|
132
|
+
- ".gitignore"
|
133
|
+
- Gemfile
|
134
|
+
- LICENSE.txt
|
135
|
+
- README.md
|
136
|
+
- Rakefile
|
137
|
+
- lib/generators/squash_repeater/install_generator.rb
|
138
|
+
- lib/generators/templates/squash_repeater_initializer.rb
|
139
|
+
- lib/squash_repeater.rb
|
140
|
+
- lib/squash_repeater/configure.rb
|
141
|
+
- lib/squash_repeater/configure/squash.rb
|
142
|
+
- lib/squash_repeater/exception_queue.rb
|
143
|
+
- lib/squash_repeater/squash_ruby.rb
|
144
|
+
- lib/squash_repeater/version.rb
|
145
|
+
- share/upstart/backburner-manager.conf
|
146
|
+
- share/upstart/backburner-worker.conf
|
147
|
+
- spec/spec_helper.rb
|
148
|
+
- spec/squash_repeater_configure_squash.rb
|
149
|
+
- spec/squash_repeater_spec.rb
|
150
|
+
- spec/squash_ruby_spec.rb
|
151
|
+
- squash_repeater-ruby.gemspec
|
152
|
+
homepage: ''
|
153
|
+
licenses:
|
154
|
+
- MIT
|
155
|
+
metadata: {}
|
156
|
+
post_install_message:
|
157
|
+
rdoc_options: []
|
158
|
+
require_paths:
|
159
|
+
- lib
|
160
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
161
|
+
requirements:
|
162
|
+
- - ">="
|
163
|
+
- !ruby/object:Gem::Version
|
164
|
+
version: '0'
|
165
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
166
|
+
requirements:
|
167
|
+
- - ">="
|
168
|
+
- !ruby/object:Gem::Version
|
169
|
+
version: '0'
|
170
|
+
requirements: []
|
171
|
+
rubyforge_project:
|
172
|
+
rubygems_version: 2.4.5
|
173
|
+
signing_key:
|
174
|
+
specification_version: 4
|
175
|
+
summary: Squash Repeater for Ruby
|
176
|
+
test_files:
|
177
|
+
- spec/spec_helper.rb
|
178
|
+
- spec/squash_repeater_configure_squash.rb
|
179
|
+
- spec/squash_repeater_spec.rb
|
180
|
+
- spec/squash_ruby_spec.rb
|
181
|
+
has_rdoc:
|