consul_stockpile 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/Dockerfile +26 -0
- data/Gemfile +4 -0
- data/README.md +39 -0
- data/Rakefile +2 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/consul_stockpile.gemspec +35 -0
- data/docker-entrypoint.sh +23 -0
- data/exe/consul_stockpile +5 -0
- data/lib/consul_stockpile/backup_consul_kv.rb +55 -0
- data/lib/consul_stockpile/backup_consul_kv_actor.rb +28 -0
- data/lib/consul_stockpile/base.rb +13 -0
- data/lib/consul_stockpile/bootstrap_consul_kv.rb +38 -0
- data/lib/consul_stockpile/bootstrap_consul_kv_actor.rb +30 -0
- data/lib/consul_stockpile/cli.rb +25 -0
- data/lib/consul_stockpile/consul_lock.rb +20 -0
- data/lib/consul_stockpile/detect_canary.rb +14 -0
- data/lib/consul_stockpile/detect_consul.rb +27 -0
- data/lib/consul_stockpile/download_backup.rb +42 -0
- data/lib/consul_stockpile/load_json_kv.rb +24 -0
- data/lib/consul_stockpile/logger.rb +25 -0
- data/lib/consul_stockpile/place_canary.rb +11 -0
- data/lib/consul_stockpile/run_stockpile.rb +74 -0
- data/lib/consul_stockpile/version.rb +3 -0
- data/lib/consul_stockpile/watch_event.rb +84 -0
- data/lib/consul_stockpile/watch_event_actor.rb +30 -0
- data/lib/consul_stockpile.rb +5 -0
- metadata +268 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: b59e8e2b4697347d0e8e63c956aac5e4533ae18f
|
4
|
+
data.tar.gz: 5c8e759867192356ac72b0900ec2e16a0e64cd52
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 91fdc2a2413a3f64435266f6a0787aa346980daf3c1ecabd6a8a58016d338329d373c3a6df241703aaef27d7aa67a34cd40c2e785d48f5c61d2bb1ccf04629a3
|
7
|
+
data.tar.gz: e778686e7bca832f2a6acca93c4d3065e5084428dbdd391c419fbcad3abdf03656606b5082d0eb045c54249ceeda76d71a4906f0ddd43b9a895b06d4934df608
|
data/.gitignore
ADDED
data/Dockerfile
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
FROM outstand/ruby-base:2.2.4-alpine
|
2
|
+
MAINTAINER Ryan Schlesinger <ryan@outstand.com>
|
3
|
+
|
4
|
+
RUN addgroup stockpile && \
|
5
|
+
adduser -S -G stockpile stockpile
|
6
|
+
|
7
|
+
RUN apk --no-cache add build-base libxml2-dev libxslt-dev openssh
|
8
|
+
|
9
|
+
ENV USE_BUNDLE_EXEC true
|
10
|
+
|
11
|
+
COPY Gemfile consul_stockpile.gemspec /consul_stockpile/
|
12
|
+
COPY lib/consul_stockpile/version.rb /consul_stockpile/lib/consul_stockpile/
|
13
|
+
RUN cd /consul_stockpile \
|
14
|
+
&& bundle config build.nokogiri --use-system-libraries \
|
15
|
+
&& bundle install \
|
16
|
+
&& git config --global push.default simple
|
17
|
+
COPY . /consul_stockpile/
|
18
|
+
RUN ln -s /consul_stockpile/exe/consul_stockpile /usr/local/bin/consul_stockpile
|
19
|
+
|
20
|
+
WORKDIR /consul_stockpile
|
21
|
+
|
22
|
+
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
23
|
+
|
24
|
+
ENV DUMB_INIT_SETSID 0
|
25
|
+
ENTRYPOINT ["/docker-entrypoint.sh"]
|
26
|
+
CMD ["consul_stockpile"]
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
# Consul Stockpile
|
2
|
+
|
3
|
+
## Installation
|
4
|
+
|
5
|
+
Add this line to your application's Gemfile:
|
6
|
+
|
7
|
+
```ruby
|
8
|
+
gem 'consul_stockpile'
|
9
|
+
```
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install consul_stockpile
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
(On AWS)
|
22
|
+
`docker run -d --net=host outstand/consul_stockpile start -b bucket -n backup`
|
23
|
+
|
24
|
+
## Development
|
25
|
+
|
26
|
+
- `docker volume create --name fog`
|
27
|
+
- `docker build -t outstand/consul_stockpile .`
|
28
|
+
- `docker run -it --rm --net=host -v $(pwd):/consul_stockpile -v fog:/fog -e FOG_LOCAL=true outstand/consul_stockpile start -b bucket -n backup`
|
29
|
+
|
30
|
+
To release a new version:
|
31
|
+
- Update the version number in `version.rb`
|
32
|
+
- Run `docker run -it --rm outstand/consul_stockpile:latest bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
33
|
+
- Run `docker build -t outstand/consul_stockpile:VERSION -f Dockerfile.release .`
|
34
|
+
- Run `docker push outstand/consul_stockpile:VERSION`
|
35
|
+
|
36
|
+
## Contributing
|
37
|
+
|
38
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/outstand/consul_stockpile.
|
39
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "consul_stockpile"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'consul_stockpile/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "consul_stockpile"
|
8
|
+
spec.version = ConsulStockpile::VERSION
|
9
|
+
spec.authors = ["Ryan Schlesinger"]
|
10
|
+
spec.email = ["ryan@outstand.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{Bootstrap and backup for consul kv store}
|
13
|
+
spec.homepage = "https://github.com/outstand/consul_stockpile"
|
14
|
+
|
15
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
16
|
+
spec.bindir = "exe"
|
17
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.add_runtime_dependency 'thor', '~> 0.19'
|
21
|
+
spec.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
|
22
|
+
spec.add_runtime_dependency 'concurrent-ruby-edge', '~> 0.2'
|
23
|
+
spec.add_runtime_dependency 'excon', '~> 0.49.0'
|
24
|
+
spec.add_runtime_dependency 'diplomat', '~> 0.17'
|
25
|
+
spec.add_runtime_dependency 'fog-aws', '~> 0.9'
|
26
|
+
spec.add_runtime_dependency 'mime-types', '~> 3.0'
|
27
|
+
|
28
|
+
spec.add_development_dependency "bundler", "~> 1.11"
|
29
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
30
|
+
spec.add_development_dependency "fog-local", "~> 0.3"
|
31
|
+
spec.add_development_dependency "rspec", "~> 3.4"
|
32
|
+
spec.add_development_dependency "fivemat", "~> 1.3"
|
33
|
+
spec.add_development_dependency "aruba", "~> 0.14"
|
34
|
+
spec.add_development_dependency "pry-byebug"
|
35
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
#!/bin/dumb-init /bin/sh
|
2
|
+
set -e
|
3
|
+
|
4
|
+
if [ -n "$USE_BUNDLE_EXEC" ]; then
|
5
|
+
STOCKPILE_BINARY="bundle exec consul_stockpile"
|
6
|
+
else
|
7
|
+
STOCKPILE_BINARY=consul_stockpile
|
8
|
+
fi
|
9
|
+
|
10
|
+
if ${STOCKPILE_BINARY} help "$1" 2>&1 | grep -q "consul_stockpile $1"; then
|
11
|
+
set -- ${STOCKPILE_BINARY} "$@"
|
12
|
+
fi
|
13
|
+
|
14
|
+
if [ "$1" = "${STOCKPILE_BINARY}" ]; then
|
15
|
+
# Enable this to run as an unprivileged user
|
16
|
+
set -- gosu stockpile "$@"
|
17
|
+
|
18
|
+
if [ -n "$FOG_LOCAL" ]; then
|
19
|
+
chown -R stockpile:stockpile /fog
|
20
|
+
fi
|
21
|
+
fi
|
22
|
+
|
23
|
+
exec "$@"
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'consul_stockpile/base'
|
2
|
+
require 'consul_stockpile/logger'
|
3
|
+
require 'fog/aws'
|
4
|
+
require 'fog/local' if ENV['FOG_LOCAL']
|
5
|
+
|
6
|
+
module ConsulStockpile
|
7
|
+
class BackupConsulKV < Base
|
8
|
+
EVENT_KEY = 'event/kv_update'.freeze
|
9
|
+
|
10
|
+
attr_accessor :bucket, :name
|
11
|
+
|
12
|
+
def initialize(bucket:, name:)
|
13
|
+
self.bucket = bucket
|
14
|
+
self.name = name
|
15
|
+
end
|
16
|
+
|
17
|
+
def call
|
18
|
+
Logger.tagged('Backup') do
|
19
|
+
Logger.info 'Starting Consul KV Backup...'
|
20
|
+
|
21
|
+
data = Diplomat::Kv.get('', {recurse: true}, :return)
|
22
|
+
if data.nil? || data.empty?
|
23
|
+
Logger.info 'Consul kv storage is empty.'
|
24
|
+
return
|
25
|
+
end
|
26
|
+
|
27
|
+
data.reject!{|kv| kv[:key].start_with?(EVENT_KEY)}
|
28
|
+
|
29
|
+
if data.empty?
|
30
|
+
Logger.info 'Consul kv storage only has blacklisted keys.'
|
31
|
+
return
|
32
|
+
end
|
33
|
+
|
34
|
+
Logger.info "Uploading consul kv backup to bucket #{self.bucket} as #{self.name}"
|
35
|
+
|
36
|
+
bucket = nil
|
37
|
+
if ENV['FOG_LOCAL']
|
38
|
+
Logger.debug 'Using fog local storage'
|
39
|
+
storage = Fog::Storage.new provider: 'Local', local_root: '/fog'
|
40
|
+
bucket = storage.directories.create(key: self.bucket)
|
41
|
+
else
|
42
|
+
storage = Fog::Storage.new provider: 'AWS', use_iam_profile: true
|
43
|
+
bucket = storage.directories.get(self.bucket)
|
44
|
+
end
|
45
|
+
|
46
|
+
bucket.files.create(
|
47
|
+
key: "#{self.name}.json",
|
48
|
+
body: data.to_json
|
49
|
+
)
|
50
|
+
|
51
|
+
Logger.info 'Backup complete.'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'concurrent'
|
2
|
+
require 'concurrent-edge'
|
3
|
+
require 'consul_stockpile/backup_consul_kv'
|
4
|
+
|
5
|
+
module ConsulStockpile
|
6
|
+
class BackupConsulKVActor < Concurrent::Actor::RestartingContext
|
7
|
+
def initialize(bucket:, name:)
|
8
|
+
@bucket = bucket
|
9
|
+
@name = name
|
10
|
+
end
|
11
|
+
|
12
|
+
def on_message(message)
|
13
|
+
if message == :backup
|
14
|
+
Logger.tagged('Backup') do
|
15
|
+
begin
|
16
|
+
BackupConsulKV.call(bucket: @bucket, name: @name)
|
17
|
+
rescue => e
|
18
|
+
Logger.warn "Warning: #{e.message}; retrying in 5 seconds"
|
19
|
+
Logger.warn e.backtrace
|
20
|
+
Concurrent::ScheduledTask.execute(5){ tell :backup }
|
21
|
+
end
|
22
|
+
end
|
23
|
+
else
|
24
|
+
pass
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'consul_stockpile/base'
|
2
|
+
require 'consul_stockpile/consul_lock'
|
3
|
+
require 'consul_stockpile/detect_canary'
|
4
|
+
require 'consul_stockpile/download_backup'
|
5
|
+
require 'consul_stockpile/load_json_kv'
|
6
|
+
require 'consul_stockpile/logger'
|
7
|
+
require 'consul_stockpile/place_canary'
|
8
|
+
|
9
|
+
module ConsulStockpile
|
10
|
+
class BootstrapConsulKV < Base
|
11
|
+
LOCK_KEY = 'stockpile/bootstrap'.freeze
|
12
|
+
|
13
|
+
attr_accessor :bucket
|
14
|
+
|
15
|
+
def initialize(bucket:)
|
16
|
+
self.bucket = bucket
|
17
|
+
end
|
18
|
+
|
19
|
+
def call
|
20
|
+
Logger.tagged('Bootstrap') do
|
21
|
+
Logger.info 'Starting Consul KV Bootstrap...'
|
22
|
+
|
23
|
+
ConsulLock.with_lock(key: LOCK_KEY) do
|
24
|
+
if DetectCanary.call.exists
|
25
|
+
Logger.info 'Canary detected; skipping bootstrap.'
|
26
|
+
return
|
27
|
+
end
|
28
|
+
|
29
|
+
json = DownloadBackup.call(bucket: bucket).json_body
|
30
|
+
LoadJsonKV.call(json: json)
|
31
|
+
PlaceCanary.call
|
32
|
+
end
|
33
|
+
|
34
|
+
Logger.info 'Bootstrap complete.'
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'concurrent'
|
2
|
+
require 'concurrent-edge'
|
3
|
+
require 'consul_stockpile/bootstrap_consul_kv'
|
4
|
+
require 'consul_stockpile/logger'
|
5
|
+
|
6
|
+
module ConsulStockpile
|
7
|
+
class BootstrapConsulKVActor < Concurrent::Actor::RestartingContext
|
8
|
+
def initialize(backup_actor:, bucket:)
|
9
|
+
@backup_actor = backup_actor
|
10
|
+
@bucket = bucket
|
11
|
+
tell :bootstrap
|
12
|
+
end
|
13
|
+
|
14
|
+
def on_message(message)
|
15
|
+
if message == :bootstrap
|
16
|
+
Logger.tagged('Bootstrap') do
|
17
|
+
begin
|
18
|
+
BootstrapConsulKV.call(bucket: @bucket)
|
19
|
+
@backup_actor << :backup
|
20
|
+
rescue => e
|
21
|
+
Logger.warn "Warning: #{e.message}; retrying in 5 seconds"
|
22
|
+
Concurrent::ScheduledTask.execute(5){ tell :bootstrap }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
else
|
26
|
+
pass
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'thor'
|
2
|
+
|
3
|
+
module ConsulStockpile
|
4
|
+
class CLI < Thor
|
5
|
+
desc 'version', 'Print out the version string'
|
6
|
+
def version
|
7
|
+
require 'consul_stockpile/version'
|
8
|
+
say ConsulStockpile::VERSION.to_s
|
9
|
+
end
|
10
|
+
|
11
|
+
desc 'start', 'Start the stockpile'
|
12
|
+
option :bucket, aliases: '-b', required: true, type: :string, banner: '<s3_bucket>'
|
13
|
+
option :name, aliases: '-n', required: true, type: :string
|
14
|
+
option :verbose, aliases: '-v', type: :boolean, default: false
|
15
|
+
def start
|
16
|
+
$stdout.sync = true
|
17
|
+
require 'consul_stockpile/run_stockpile'
|
18
|
+
RunStockpile.call(
|
19
|
+
bucket: options[:bucket],
|
20
|
+
name: options[:name],
|
21
|
+
verbose: options[:verbose]
|
22
|
+
)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module ConsulStockpile
|
2
|
+
module ConsulLock
|
3
|
+
extend self
|
4
|
+
|
5
|
+
def with_lock(key:)
|
6
|
+
sessionid = nil
|
7
|
+
locked = false
|
8
|
+
sessionid = Diplomat::Session.create(Name: "#{key}.lock")
|
9
|
+
locked = Diplomat::Lock.wait_to_acquire(key, sessionid)
|
10
|
+
|
11
|
+
yield
|
12
|
+
ensure
|
13
|
+
if sessionid != nil
|
14
|
+
Diplomat::Lock.release(key, sessionid) if locked
|
15
|
+
Diplomat::Session.destroy(sessionid)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'consul_stockpile/base'
|
2
|
+
require 'excon'
|
3
|
+
|
4
|
+
module ConsulStockpile
|
5
|
+
class DetectConsul < Base
|
6
|
+
URL = 'http://127.0.0.1:8500/v1/agent/self'.freeze
|
7
|
+
|
8
|
+
def call
|
9
|
+
running = true
|
10
|
+
begin
|
11
|
+
Excon.get(
|
12
|
+
URL,
|
13
|
+
expects: [200],
|
14
|
+
connect_timeout: 5,
|
15
|
+
read_timeout: 5,
|
16
|
+
write_timeout: 5,
|
17
|
+
tcp_nodelay: true
|
18
|
+
)
|
19
|
+
rescue Excon::Errors::SocketError, Excon::Errors::HTTPStatusError
|
20
|
+
running = false
|
21
|
+
end
|
22
|
+
OpenStruct.new(
|
23
|
+
running: running
|
24
|
+
)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'consul_stockpile/base'
|
2
|
+
require 'consul_stockpile/logger'
|
3
|
+
require 'fog/aws'
|
4
|
+
require 'fog/local' if ENV['FOG_LOCAL']
|
5
|
+
|
6
|
+
module ConsulStockpile
|
7
|
+
class DownloadBackup < Base
|
8
|
+
attr_accessor :bucket
|
9
|
+
|
10
|
+
def initialize(bucket:)
|
11
|
+
self.bucket = bucket
|
12
|
+
end
|
13
|
+
|
14
|
+
def call
|
15
|
+
Logger.info "Downloading consul kv backup from bucket #{self.bucket}"
|
16
|
+
|
17
|
+
bucket = nil
|
18
|
+
if ENV['FOG_LOCAL']
|
19
|
+
Logger.debug 'Using fog local storage'
|
20
|
+
storage = Fog::Storage.new provider: 'Local', local_root: '/fog'
|
21
|
+
bucket = storage.directories.create(key: self.bucket)
|
22
|
+
else
|
23
|
+
storage = Fog::Storage.new provider: 'AWS', use_iam_profile: true
|
24
|
+
bucket = storage.directories.get(self.bucket)
|
25
|
+
end
|
26
|
+
|
27
|
+
body = nil
|
28
|
+
last_modified = nil
|
29
|
+
bucket.files.each do |file|
|
30
|
+
next if last_modified != nil && file.last_modified < last_modified
|
31
|
+
last_modified = file.last_modified
|
32
|
+
body = file.body
|
33
|
+
end
|
34
|
+
|
35
|
+
Logger.info 'Download complete.'
|
36
|
+
|
37
|
+
OpenStruct.new(
|
38
|
+
json_body: body
|
39
|
+
)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'consul_stockpile/base'
|
2
|
+
require 'consul_stockpile/logger'
|
3
|
+
|
4
|
+
module ConsulStockpile
|
5
|
+
class LoadJsonKV < Base
|
6
|
+
attr_accessor :json
|
7
|
+
|
8
|
+
def initialize(json:)
|
9
|
+
self.json = json
|
10
|
+
end
|
11
|
+
|
12
|
+
def call
|
13
|
+
Logger.info 'Loading json backup into consul kv store'
|
14
|
+
|
15
|
+
return if json.nil?
|
16
|
+
ary = JSON.parse(json)
|
17
|
+
return if ary.empty?
|
18
|
+
|
19
|
+
ary.each do |item|
|
20
|
+
Diplomat::Kv.put(item['key'], item['value'])
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module ConsulStockpile
|
4
|
+
class Logger
|
5
|
+
class << self
|
6
|
+
%i[debug info warn error fatal].each do |sym|
|
7
|
+
define_method(sym) do |message|
|
8
|
+
logger.send(sym, "#{Thread.current[:logger_tag]}#{message}")
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.tagged(tag)
|
14
|
+
existing_tag = Thread.current[:logger_tag]
|
15
|
+
Thread.current[:logger_tag] = "[#{tag}] "
|
16
|
+
yield
|
17
|
+
ensure
|
18
|
+
Thread.current[:logger_tag] = existing_tag
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.logger
|
22
|
+
Thread.current[:logger] = ::Logger.new(STDOUT)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'consul_stockpile/base'
|
2
|
+
require 'consul_stockpile/logger'
|
3
|
+
require 'concurrent'
|
4
|
+
require 'consul_stockpile/detect_consul'
|
5
|
+
require 'consul_stockpile/bootstrap_consul_kv_actor'
|
6
|
+
require 'consul_stockpile/watch_event_actor'
|
7
|
+
require 'consul_stockpile/backup_consul_kv_actor'
|
8
|
+
|
9
|
+
module ConsulStockpile
|
10
|
+
class RunStockpile < Base
|
11
|
+
attr_accessor :bucket, :name, :verbose
|
12
|
+
|
13
|
+
def initialize(bucket:, name:, verbose: false)
|
14
|
+
self.bucket = bucket
|
15
|
+
self.name = name
|
16
|
+
self.verbose = verbose
|
17
|
+
end
|
18
|
+
|
19
|
+
def call
|
20
|
+
Concurrent.use_stdlib_logger(Logger::DEBUG) if self.verbose
|
21
|
+
|
22
|
+
self_read, self_write = IO.pipe
|
23
|
+
%w(INT TERM).each do |sig|
|
24
|
+
begin
|
25
|
+
trap sig do
|
26
|
+
self_write.puts(sig)
|
27
|
+
end
|
28
|
+
rescue ArgumentError
|
29
|
+
puts "Signal #{sig} not supported"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
begin
|
34
|
+
while !DetectConsul.call.running
|
35
|
+
Logger.warn 'Local consul agent not detected, sleeping for 5 seconds'
|
36
|
+
sleep 5
|
37
|
+
end
|
38
|
+
|
39
|
+
backup_actor = BackupConsulKVActor.spawn(
|
40
|
+
:backup_consul_kv,
|
41
|
+
bucket: self.bucket,
|
42
|
+
name: self.name
|
43
|
+
)
|
44
|
+
|
45
|
+
BootstrapConsulKVActor.spawn(
|
46
|
+
:bootstrap_consul_kv,
|
47
|
+
backup_actor: backup_actor,
|
48
|
+
bucket: self.bucket
|
49
|
+
)
|
50
|
+
WatchEventActor.spawn(:watch_event, backup_actor: backup_actor)
|
51
|
+
|
52
|
+
|
53
|
+
while readable_io = IO.select([self_read])
|
54
|
+
signal = readable_io.first[0].gets.strip
|
55
|
+
handle_signal(signal)
|
56
|
+
end
|
57
|
+
|
58
|
+
rescue Interrupt
|
59
|
+
Logger.info 'Exiting'
|
60
|
+
# actors are cleaned up in at_exit handler
|
61
|
+
exit 0
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def handle_signal(sig)
|
66
|
+
case sig
|
67
|
+
when 'INT'
|
68
|
+
raise Interrupt
|
69
|
+
when 'TERM'
|
70
|
+
raise Interrupt
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'consul_stockpile/base'
|
2
|
+
require 'excon'
|
3
|
+
require 'json'
|
4
|
+
require 'diplomat'
|
5
|
+
require 'consul_stockpile/consul_lock'
|
6
|
+
require 'consul_stockpile/logger'
|
7
|
+
|
8
|
+
module ConsulStockpile
|
9
|
+
class WatchEvent < Base
|
10
|
+
EVENT = 'kv_update'.freeze
|
11
|
+
URL = 'http://127.0.0.1:8500/v1/event/list'.freeze
|
12
|
+
KEY = 'event/kv_update'.freeze
|
13
|
+
LOCK_KEY = 'event/kv_update/lock'.freeze
|
14
|
+
|
15
|
+
attr_accessor :handler
|
16
|
+
|
17
|
+
def initialize(handler:)
|
18
|
+
self.handler = handler
|
19
|
+
end
|
20
|
+
|
21
|
+
def call
|
22
|
+
Logger.tagged('WatchEvent') do
|
23
|
+
Logger.info "Watching for event: #{EVENT}"
|
24
|
+
|
25
|
+
response = Excon.get(
|
26
|
+
URL,
|
27
|
+
query: {name: EVENT},
|
28
|
+
expects: [200],
|
29
|
+
connect_timeout: 5,
|
30
|
+
read_timeout: 5,
|
31
|
+
write_timeout: 5,
|
32
|
+
tcp_nodelay: true
|
33
|
+
)
|
34
|
+
|
35
|
+
handle_events(response.body)
|
36
|
+
|
37
|
+
loop do
|
38
|
+
index = response.headers['X-Consul-Index']
|
39
|
+
response = Excon.get(
|
40
|
+
URL,
|
41
|
+
query: {name: EVENT, index: index},
|
42
|
+
expects: [200],
|
43
|
+
connect_timeout: 5,
|
44
|
+
read_timeout: 86400,
|
45
|
+
write_timeout: 5,
|
46
|
+
tcp_nodelay: true
|
47
|
+
)
|
48
|
+
|
49
|
+
handle_events(response.body)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
def handle_events(events)
|
56
|
+
events = JSON.parse(events)
|
57
|
+
return if events.empty?
|
58
|
+
|
59
|
+
sift(events) do |event|
|
60
|
+
handler.call(event)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def sift(events)
|
65
|
+
ConsulLock.with_lock(key: LOCK_KEY) do
|
66
|
+
last_worked = Diplomat::Kv.get(KEY, {}, :return)
|
67
|
+
# Work around diplomat returning "" instead of nil
|
68
|
+
last_worked = nil if last_worked.empty?
|
69
|
+
last_worked = last_worked.to_i unless last_worked.nil?
|
70
|
+
|
71
|
+
event = events.last
|
72
|
+
Logger.info "Sifting event: #{event.inspect}"
|
73
|
+
ltime = event['LTime']
|
74
|
+
if last_worked.nil? || ltime > last_worked
|
75
|
+
yield event
|
76
|
+
|
77
|
+
Diplomat::Kv.put(KEY, ltime.to_s)
|
78
|
+
else
|
79
|
+
Logger.info 'Skipping duplicate event'
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'concurrent'
|
2
|
+
require 'concurrent-edge'
|
3
|
+
require 'consul_stockpile/watch_event'
|
4
|
+
|
5
|
+
module ConsulStockpile
|
6
|
+
class WatchEventActor < Concurrent::Actor::RestartingContext
|
7
|
+
def initialize(backup_actor:)
|
8
|
+
@backup_actor = backup_actor
|
9
|
+
tell :watch
|
10
|
+
end
|
11
|
+
|
12
|
+
def on_message(message)
|
13
|
+
if message == :watch
|
14
|
+
Logger.tagged('WatchEvent') do
|
15
|
+
begin
|
16
|
+
WatchEvent.call(
|
17
|
+
handler: ->(event){ @backup_actor << :backup }
|
18
|
+
)
|
19
|
+
rescue => e
|
20
|
+
Logger.warn "Warning: #{e.message}; retrying in 5 seconds"
|
21
|
+
Logger.warn e.backtrace
|
22
|
+
Concurrent::ScheduledTask.execute(5){ tell :watch }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
else
|
26
|
+
pass
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
metadata
ADDED
@@ -0,0 +1,268 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: consul_stockpile
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ryan Schlesinger
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-05-11 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: thor
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.19'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0.19'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: concurrent-ruby
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: concurrent-ruby-edge
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0.2'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0.2'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: excon
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.49.0
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.49.0
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: diplomat
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0.17'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0.17'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: fog-aws
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0.9'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0.9'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: mime-types
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '3.0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '3.0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: bundler
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '1.11'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '1.11'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: rake
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - "~>"
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '10.0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - "~>"
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '10.0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: fog-local
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0.3'
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0.3'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: rspec
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '3.4'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '3.4'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: fivemat
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - "~>"
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '1.3'
|
174
|
+
type: :development
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - "~>"
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '1.3'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: aruba
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - "~>"
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0.14'
|
188
|
+
type: :development
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - "~>"
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '0.14'
|
195
|
+
- !ruby/object:Gem::Dependency
|
196
|
+
name: pry-byebug
|
197
|
+
requirement: !ruby/object:Gem::Requirement
|
198
|
+
requirements:
|
199
|
+
- - ">="
|
200
|
+
- !ruby/object:Gem::Version
|
201
|
+
version: '0'
|
202
|
+
type: :development
|
203
|
+
prerelease: false
|
204
|
+
version_requirements: !ruby/object:Gem::Requirement
|
205
|
+
requirements:
|
206
|
+
- - ">="
|
207
|
+
- !ruby/object:Gem::Version
|
208
|
+
version: '0'
|
209
|
+
description:
|
210
|
+
email:
|
211
|
+
- ryan@outstand.com
|
212
|
+
executables:
|
213
|
+
- consul_stockpile
|
214
|
+
extensions: []
|
215
|
+
extra_rdoc_files: []
|
216
|
+
files:
|
217
|
+
- ".gitignore"
|
218
|
+
- Dockerfile
|
219
|
+
- Gemfile
|
220
|
+
- README.md
|
221
|
+
- Rakefile
|
222
|
+
- bin/console
|
223
|
+
- bin/setup
|
224
|
+
- consul_stockpile.gemspec
|
225
|
+
- docker-entrypoint.sh
|
226
|
+
- exe/consul_stockpile
|
227
|
+
- lib/consul_stockpile.rb
|
228
|
+
- lib/consul_stockpile/backup_consul_kv.rb
|
229
|
+
- lib/consul_stockpile/backup_consul_kv_actor.rb
|
230
|
+
- lib/consul_stockpile/base.rb
|
231
|
+
- lib/consul_stockpile/bootstrap_consul_kv.rb
|
232
|
+
- lib/consul_stockpile/bootstrap_consul_kv_actor.rb
|
233
|
+
- lib/consul_stockpile/cli.rb
|
234
|
+
- lib/consul_stockpile/consul_lock.rb
|
235
|
+
- lib/consul_stockpile/detect_canary.rb
|
236
|
+
- lib/consul_stockpile/detect_consul.rb
|
237
|
+
- lib/consul_stockpile/download_backup.rb
|
238
|
+
- lib/consul_stockpile/load_json_kv.rb
|
239
|
+
- lib/consul_stockpile/logger.rb
|
240
|
+
- lib/consul_stockpile/place_canary.rb
|
241
|
+
- lib/consul_stockpile/run_stockpile.rb
|
242
|
+
- lib/consul_stockpile/version.rb
|
243
|
+
- lib/consul_stockpile/watch_event.rb
|
244
|
+
- lib/consul_stockpile/watch_event_actor.rb
|
245
|
+
homepage: https://github.com/outstand/consul_stockpile
|
246
|
+
licenses: []
|
247
|
+
metadata: {}
|
248
|
+
post_install_message:
|
249
|
+
rdoc_options: []
|
250
|
+
require_paths:
|
251
|
+
- lib
|
252
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
253
|
+
requirements:
|
254
|
+
- - ">="
|
255
|
+
- !ruby/object:Gem::Version
|
256
|
+
version: '0'
|
257
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
258
|
+
requirements:
|
259
|
+
- - ">="
|
260
|
+
- !ruby/object:Gem::Version
|
261
|
+
version: '0'
|
262
|
+
requirements: []
|
263
|
+
rubyforge_project:
|
264
|
+
rubygems_version: 2.6.3
|
265
|
+
signing_key:
|
266
|
+
specification_version: 4
|
267
|
+
summary: Bootstrap and backup for consul kv store
|
268
|
+
test_files: []
|