consul_stockpile 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: b59e8e2b4697347d0e8e63c956aac5e4533ae18f
4
+ data.tar.gz: 5c8e759867192356ac72b0900ec2e16a0e64cd52
5
+ SHA512:
6
+ metadata.gz: 91fdc2a2413a3f64435266f6a0787aa346980daf3c1ecabd6a8a58016d338329d373c3a6df241703aaef27d7aa67a34cd40c2e785d48f5c61d2bb1ccf04629a3
7
+ data.tar.gz: e778686e7bca832f2a6acca93c4d3065e5084428dbdd391c419fbcad3abdf03656606b5082d0eb045c54249ceeda76d71a4906f0ddd43b9a895b06d4934df608
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/Dockerfile ADDED
@@ -0,0 +1,26 @@
1
+ FROM outstand/ruby-base:2.2.4-alpine
2
+ MAINTAINER Ryan Schlesinger <ryan@outstand.com>
3
+
4
+ RUN addgroup stockpile && \
5
+ adduser -S -G stockpile stockpile
6
+
7
+ RUN apk --no-cache add build-base libxml2-dev libxslt-dev openssh
8
+
9
+ ENV USE_BUNDLE_EXEC true
10
+
11
+ COPY Gemfile consul_stockpile.gemspec /consul_stockpile/
12
+ COPY lib/consul_stockpile/version.rb /consul_stockpile/lib/consul_stockpile/
13
+ RUN cd /consul_stockpile \
14
+ && bundle config build.nokogiri --use-system-libraries \
15
+ && bundle install \
16
+ && git config --global push.default simple
17
+ COPY . /consul_stockpile/
18
+ RUN ln -s /consul_stockpile/exe/consul_stockpile /usr/local/bin/consul_stockpile
19
+
20
+ WORKDIR /consul_stockpile
21
+
22
+ COPY docker-entrypoint.sh /docker-entrypoint.sh
23
+
24
+ ENV DUMB_INIT_SETSID 0
25
+ ENTRYPOINT ["/docker-entrypoint.sh"]
26
+ CMD ["consul_stockpile"]
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in consul_stockpile.gemspec
4
+ gemspec
data/README.md ADDED
@@ -0,0 +1,39 @@
1
+ # Consul Stockpile
2
+
3
+ ## Installation
4
+
5
+ Add this line to your application's Gemfile:
6
+
7
+ ```ruby
8
+ gem 'consul_stockpile'
9
+ ```
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install consul_stockpile
18
+
19
+ ## Usage
20
+
21
+ (On AWS)
22
+ `docker run -d --net=host outstand/consul_stockpile start -b bucket -n backup`
23
+
24
+ ## Development
25
+
26
+ - `docker volume create --name fog`
27
+ - `docker build -t outstand/consul_stockpile .`
28
+ - `docker run -it --rm --net=host -v $(pwd):/consul_stockpile -v fog:/fog -e FOG_LOCAL=true outstand/consul_stockpile start -b bucket -n backup`
29
+
30
+ To release a new version:
31
+ - Update the version number in `version.rb`
32
+ - Run `docker run -it --rm outstand/consul_stockpile:latest bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
33
+ - Run `docker build -t outstand/consul_stockpile:VERSION -f Dockerfile.release .`
34
+ - Run `docker push outstand/consul_stockpile:VERSION`
35
+
36
+ ## Contributing
37
+
38
+ Bug reports and pull requests are welcome on GitHub at https://github.com/outstand/consul_stockpile.
39
+
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require "bundler/gem_tasks"
2
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "consul_stockpile"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,35 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'consul_stockpile/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "consul_stockpile"
8
+ spec.version = ConsulStockpile::VERSION
9
+ spec.authors = ["Ryan Schlesinger"]
10
+ spec.email = ["ryan@outstand.com"]
11
+
12
+ spec.summary = %q{Bootstrap and backup for consul kv store}
13
+ spec.homepage = "https://github.com/outstand/consul_stockpile"
14
+
15
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
16
+ spec.bindir = "exe"
17
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_runtime_dependency 'thor', '~> 0.19'
21
+ spec.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
22
+ spec.add_runtime_dependency 'concurrent-ruby-edge', '~> 0.2'
23
+ spec.add_runtime_dependency 'excon', '~> 0.49.0'
24
+ spec.add_runtime_dependency 'diplomat', '~> 0.17'
25
+ spec.add_runtime_dependency 'fog-aws', '~> 0.9'
26
+ spec.add_runtime_dependency 'mime-types', '~> 3.0'
27
+
28
+ spec.add_development_dependency "bundler", "~> 1.11"
29
+ spec.add_development_dependency "rake", "~> 10.0"
30
+ spec.add_development_dependency "fog-local", "~> 0.3"
31
+ spec.add_development_dependency "rspec", "~> 3.4"
32
+ spec.add_development_dependency "fivemat", "~> 1.3"
33
+ spec.add_development_dependency "aruba", "~> 0.14"
34
+ spec.add_development_dependency "pry-byebug"
35
+ end
@@ -0,0 +1,23 @@
1
+ #!/bin/dumb-init /bin/sh
2
+ set -e
3
+
4
+ if [ -n "$USE_BUNDLE_EXEC" ]; then
5
+ STOCKPILE_BINARY="bundle exec consul_stockpile"
6
+ else
7
+ STOCKPILE_BINARY=consul_stockpile
8
+ fi
9
+
10
+ if ${STOCKPILE_BINARY} help "$1" 2>&1 | grep -q "consul_stockpile $1"; then
11
+ set -- ${STOCKPILE_BINARY} "$@"
12
+ fi
13
+
14
+ if [ "$1" = "${STOCKPILE_BINARY}" ]; then
15
+ # Enable this to run as an unprivileged user
16
+ set -- gosu stockpile "$@"
17
+
18
+ if [ -n "$FOG_LOCAL" ]; then
19
+ chown -R stockpile:stockpile /fog
20
+ fi
21
+ fi
22
+
23
+ exec "$@"
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'consul_stockpile/cli'
4
+
5
+ ConsulStockpile::CLI.start
@@ -0,0 +1,55 @@
1
+ require 'consul_stockpile/base'
2
+ require 'consul_stockpile/logger'
3
+ require 'fog/aws'
4
+ require 'fog/local' if ENV['FOG_LOCAL']
5
+
6
+ module ConsulStockpile
7
+ class BackupConsulKV < Base
8
+ EVENT_KEY = 'event/kv_update'.freeze
9
+
10
+ attr_accessor :bucket, :name
11
+
12
+ def initialize(bucket:, name:)
13
+ self.bucket = bucket
14
+ self.name = name
15
+ end
16
+
17
+ def call
18
+ Logger.tagged('Backup') do
19
+ Logger.info 'Starting Consul KV Backup...'
20
+
21
+ data = Diplomat::Kv.get('', {recurse: true}, :return)
22
+ if data.nil? || data.empty?
23
+ Logger.info 'Consul kv storage is empty.'
24
+ return
25
+ end
26
+
27
+ data.reject!{|kv| kv[:key].start_with?(EVENT_KEY)}
28
+
29
+ if data.empty?
30
+ Logger.info 'Consul kv storage only has blacklisted keys.'
31
+ return
32
+ end
33
+
34
+ Logger.info "Uploading consul kv backup to bucket #{self.bucket} as #{self.name}"
35
+
36
+ bucket = nil
37
+ if ENV['FOG_LOCAL']
38
+ Logger.debug 'Using fog local storage'
39
+ storage = Fog::Storage.new provider: 'Local', local_root: '/fog'
40
+ bucket = storage.directories.create(key: self.bucket)
41
+ else
42
+ storage = Fog::Storage.new provider: 'AWS', use_iam_profile: true
43
+ bucket = storage.directories.get(self.bucket)
44
+ end
45
+
46
+ bucket.files.create(
47
+ key: "#{self.name}.json",
48
+ body: data.to_json
49
+ )
50
+
51
+ Logger.info 'Backup complete.'
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,28 @@
1
+ require 'concurrent'
2
+ require 'concurrent-edge'
3
+ require 'consul_stockpile/backup_consul_kv'
4
+
5
+ module ConsulStockpile
6
+ class BackupConsulKVActor < Concurrent::Actor::RestartingContext
7
+ def initialize(bucket:, name:)
8
+ @bucket = bucket
9
+ @name = name
10
+ end
11
+
12
+ def on_message(message)
13
+ if message == :backup
14
+ Logger.tagged('Backup') do
15
+ begin
16
+ BackupConsulKV.call(bucket: @bucket, name: @name)
17
+ rescue => e
18
+ Logger.warn "Warning: #{e.message}; retrying in 5 seconds"
19
+ Logger.warn e.backtrace
20
+ Concurrent::ScheduledTask.execute(5){ tell :backup }
21
+ end
22
+ end
23
+ else
24
+ pass
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,13 @@
1
+ require 'ostruct'
2
+
3
+ module ConsulStockpile
4
+ class Base
5
+ def self.call(*args)
6
+ new(*args).call
7
+ end
8
+
9
+ def call
10
+ OpenStruct.new()
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,38 @@
1
+ require 'consul_stockpile/base'
2
+ require 'consul_stockpile/consul_lock'
3
+ require 'consul_stockpile/detect_canary'
4
+ require 'consul_stockpile/download_backup'
5
+ require 'consul_stockpile/load_json_kv'
6
+ require 'consul_stockpile/logger'
7
+ require 'consul_stockpile/place_canary'
8
+
9
+ module ConsulStockpile
10
+ class BootstrapConsulKV < Base
11
+ LOCK_KEY = 'stockpile/bootstrap'.freeze
12
+
13
+ attr_accessor :bucket
14
+
15
+ def initialize(bucket:)
16
+ self.bucket = bucket
17
+ end
18
+
19
+ def call
20
+ Logger.tagged('Bootstrap') do
21
+ Logger.info 'Starting Consul KV Bootstrap...'
22
+
23
+ ConsulLock.with_lock(key: LOCK_KEY) do
24
+ if DetectCanary.call.exists
25
+ Logger.info 'Canary detected; skipping bootstrap.'
26
+ return
27
+ end
28
+
29
+ json = DownloadBackup.call(bucket: bucket).json_body
30
+ LoadJsonKV.call(json: json)
31
+ PlaceCanary.call
32
+ end
33
+
34
+ Logger.info 'Bootstrap complete.'
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,30 @@
1
+ require 'concurrent'
2
+ require 'concurrent-edge'
3
+ require 'consul_stockpile/bootstrap_consul_kv'
4
+ require 'consul_stockpile/logger'
5
+
6
+ module ConsulStockpile
7
+ class BootstrapConsulKVActor < Concurrent::Actor::RestartingContext
8
+ def initialize(backup_actor:, bucket:)
9
+ @backup_actor = backup_actor
10
+ @bucket = bucket
11
+ tell :bootstrap
12
+ end
13
+
14
+ def on_message(message)
15
+ if message == :bootstrap
16
+ Logger.tagged('Bootstrap') do
17
+ begin
18
+ BootstrapConsulKV.call(bucket: @bucket)
19
+ @backup_actor << :backup
20
+ rescue => e
21
+ Logger.warn "Warning: #{e.message}; retrying in 5 seconds"
22
+ Concurrent::ScheduledTask.execute(5){ tell :bootstrap }
23
+ end
24
+ end
25
+ else
26
+ pass
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,25 @@
1
+ require 'thor'
2
+
3
+ module ConsulStockpile
4
+ class CLI < Thor
5
+ desc 'version', 'Print out the version string'
6
+ def version
7
+ require 'consul_stockpile/version'
8
+ say ConsulStockpile::VERSION.to_s
9
+ end
10
+
11
+ desc 'start', 'Start the stockpile'
12
+ option :bucket, aliases: '-b', required: true, type: :string, banner: '<s3_bucket>'
13
+ option :name, aliases: '-n', required: true, type: :string
14
+ option :verbose, aliases: '-v', type: :boolean, default: false
15
+ def start
16
+ $stdout.sync = true
17
+ require 'consul_stockpile/run_stockpile'
18
+ RunStockpile.call(
19
+ bucket: options[:bucket],
20
+ name: options[:name],
21
+ verbose: options[:verbose]
22
+ )
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,20 @@
1
+ module ConsulStockpile
2
+ module ConsulLock
3
+ extend self
4
+
5
+ def with_lock(key:)
6
+ sessionid = nil
7
+ locked = false
8
+ sessionid = Diplomat::Session.create(Name: "#{key}.lock")
9
+ locked = Diplomat::Lock.wait_to_acquire(key, sessionid)
10
+
11
+ yield
12
+ ensure
13
+ if sessionid != nil
14
+ Diplomat::Lock.release(key, sessionid) if locked
15
+ Diplomat::Session.destroy(sessionid)
16
+ end
17
+ end
18
+
19
+ end
20
+ end
@@ -0,0 +1,14 @@
1
+ require 'consul_stockpile/base'
2
+
3
+ module ConsulStockpile
4
+ class DetectCanary < Base
5
+ KEY = 'stockpile/canary'.freeze
6
+
7
+ def call
8
+ canary = Diplomat::Kv.get(KEY, {}, :return)
9
+ OpenStruct.new(
10
+ exists: canary != ""
11
+ )
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,27 @@
1
+ require 'consul_stockpile/base'
2
+ require 'excon'
3
+
4
+ module ConsulStockpile
5
+ class DetectConsul < Base
6
+ URL = 'http://127.0.0.1:8500/v1/agent/self'.freeze
7
+
8
+ def call
9
+ running = true
10
+ begin
11
+ Excon.get(
12
+ URL,
13
+ expects: [200],
14
+ connect_timeout: 5,
15
+ read_timeout: 5,
16
+ write_timeout: 5,
17
+ tcp_nodelay: true
18
+ )
19
+ rescue Excon::Errors::SocketError, Excon::Errors::HTTPStatusError
20
+ running = false
21
+ end
22
+ OpenStruct.new(
23
+ running: running
24
+ )
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,42 @@
1
+ require 'consul_stockpile/base'
2
+ require 'consul_stockpile/logger'
3
+ require 'fog/aws'
4
+ require 'fog/local' if ENV['FOG_LOCAL']
5
+
6
+ module ConsulStockpile
7
+ class DownloadBackup < Base
8
+ attr_accessor :bucket
9
+
10
+ def initialize(bucket:)
11
+ self.bucket = bucket
12
+ end
13
+
14
+ def call
15
+ Logger.info "Downloading consul kv backup from bucket #{self.bucket}"
16
+
17
+ bucket = nil
18
+ if ENV['FOG_LOCAL']
19
+ Logger.debug 'Using fog local storage'
20
+ storage = Fog::Storage.new provider: 'Local', local_root: '/fog'
21
+ bucket = storage.directories.create(key: self.bucket)
22
+ else
23
+ storage = Fog::Storage.new provider: 'AWS', use_iam_profile: true
24
+ bucket = storage.directories.get(self.bucket)
25
+ end
26
+
27
+ body = nil
28
+ last_modified = nil
29
+ bucket.files.each do |file|
30
+ next if last_modified != nil && file.last_modified < last_modified
31
+ last_modified = file.last_modified
32
+ body = file.body
33
+ end
34
+
35
+ Logger.info 'Download complete.'
36
+
37
+ OpenStruct.new(
38
+ json_body: body
39
+ )
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,24 @@
1
+ require 'consul_stockpile/base'
2
+ require 'consul_stockpile/logger'
3
+
4
+ module ConsulStockpile
5
+ class LoadJsonKV < Base
6
+ attr_accessor :json
7
+
8
+ def initialize(json:)
9
+ self.json = json
10
+ end
11
+
12
+ def call
13
+ Logger.info 'Loading json backup into consul kv store'
14
+
15
+ return if json.nil?
16
+ ary = JSON.parse(json)
17
+ return if ary.empty?
18
+
19
+ ary.each do |item|
20
+ Diplomat::Kv.put(item['key'], item['value'])
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,25 @@
1
+ require 'logger'
2
+
3
+ module ConsulStockpile
4
+ class Logger
5
+ class << self
6
+ %i[debug info warn error fatal].each do |sym|
7
+ define_method(sym) do |message|
8
+ logger.send(sym, "#{Thread.current[:logger_tag]}#{message}")
9
+ end
10
+ end
11
+ end
12
+
13
+ def self.tagged(tag)
14
+ existing_tag = Thread.current[:logger_tag]
15
+ Thread.current[:logger_tag] = "[#{tag}] "
16
+ yield
17
+ ensure
18
+ Thread.current[:logger_tag] = existing_tag
19
+ end
20
+
21
+ def self.logger
22
+ Thread.current[:logger] = ::Logger.new(STDOUT)
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,11 @@
1
+ require 'consul_stockpile/base'
2
+
3
+ module ConsulStockpile
4
+ class PlaceCanary < Base
5
+ KEY = 'stockpile/canary'.freeze
6
+
7
+ def call
8
+ Diplomat::Kv.put(KEY, 'tweet'.freeze)
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,74 @@
1
+ require 'consul_stockpile/base'
2
+ require 'consul_stockpile/logger'
3
+ require 'concurrent'
4
+ require 'consul_stockpile/detect_consul'
5
+ require 'consul_stockpile/bootstrap_consul_kv_actor'
6
+ require 'consul_stockpile/watch_event_actor'
7
+ require 'consul_stockpile/backup_consul_kv_actor'
8
+
9
+ module ConsulStockpile
10
+ class RunStockpile < Base
11
+ attr_accessor :bucket, :name, :verbose
12
+
13
+ def initialize(bucket:, name:, verbose: false)
14
+ self.bucket = bucket
15
+ self.name = name
16
+ self.verbose = verbose
17
+ end
18
+
19
+ def call
20
+ Concurrent.use_stdlib_logger(Logger::DEBUG) if self.verbose
21
+
22
+ self_read, self_write = IO.pipe
23
+ %w(INT TERM).each do |sig|
24
+ begin
25
+ trap sig do
26
+ self_write.puts(sig)
27
+ end
28
+ rescue ArgumentError
29
+ puts "Signal #{sig} not supported"
30
+ end
31
+ end
32
+
33
+ begin
34
+ while !DetectConsul.call.running
35
+ Logger.warn 'Local consul agent not detected, sleeping for 5 seconds'
36
+ sleep 5
37
+ end
38
+
39
+ backup_actor = BackupConsulKVActor.spawn(
40
+ :backup_consul_kv,
41
+ bucket: self.bucket,
42
+ name: self.name
43
+ )
44
+
45
+ BootstrapConsulKVActor.spawn(
46
+ :bootstrap_consul_kv,
47
+ backup_actor: backup_actor,
48
+ bucket: self.bucket
49
+ )
50
+ WatchEventActor.spawn(:watch_event, backup_actor: backup_actor)
51
+
52
+
53
+ while readable_io = IO.select([self_read])
54
+ signal = readable_io.first[0].gets.strip
55
+ handle_signal(signal)
56
+ end
57
+
58
+ rescue Interrupt
59
+ Logger.info 'Exiting'
60
+ # actors are cleaned up in at_exit handler
61
+ exit 0
62
+ end
63
+ end
64
+
65
+ def handle_signal(sig)
66
+ case sig
67
+ when 'INT'
68
+ raise Interrupt
69
+ when 'TERM'
70
+ raise Interrupt
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,3 @@
1
+ module ConsulStockpile
2
+ VERSION = "0.1.0"
3
+ end
@@ -0,0 +1,84 @@
1
+ require 'consul_stockpile/base'
2
+ require 'excon'
3
+ require 'json'
4
+ require 'diplomat'
5
+ require 'consul_stockpile/consul_lock'
6
+ require 'consul_stockpile/logger'
7
+
8
+ module ConsulStockpile
9
+ class WatchEvent < Base
10
+ EVENT = 'kv_update'.freeze
11
+ URL = 'http://127.0.0.1:8500/v1/event/list'.freeze
12
+ KEY = 'event/kv_update'.freeze
13
+ LOCK_KEY = 'event/kv_update/lock'.freeze
14
+
15
+ attr_accessor :handler
16
+
17
+ def initialize(handler:)
18
+ self.handler = handler
19
+ end
20
+
21
+ def call
22
+ Logger.tagged('WatchEvent') do
23
+ Logger.info "Watching for event: #{EVENT}"
24
+
25
+ response = Excon.get(
26
+ URL,
27
+ query: {name: EVENT},
28
+ expects: [200],
29
+ connect_timeout: 5,
30
+ read_timeout: 5,
31
+ write_timeout: 5,
32
+ tcp_nodelay: true
33
+ )
34
+
35
+ handle_events(response.body)
36
+
37
+ loop do
38
+ index = response.headers['X-Consul-Index']
39
+ response = Excon.get(
40
+ URL,
41
+ query: {name: EVENT, index: index},
42
+ expects: [200],
43
+ connect_timeout: 5,
44
+ read_timeout: 86400,
45
+ write_timeout: 5,
46
+ tcp_nodelay: true
47
+ )
48
+
49
+ handle_events(response.body)
50
+ end
51
+ end
52
+ end
53
+
54
+ private
55
+ def handle_events(events)
56
+ events = JSON.parse(events)
57
+ return if events.empty?
58
+
59
+ sift(events) do |event|
60
+ handler.call(event)
61
+ end
62
+ end
63
+
64
+ def sift(events)
65
+ ConsulLock.with_lock(key: LOCK_KEY) do
66
+ last_worked = Diplomat::Kv.get(KEY, {}, :return)
67
+ # Work around diplomat returning "" instead of nil
68
+ last_worked = nil if last_worked.empty?
69
+ last_worked = last_worked.to_i unless last_worked.nil?
70
+
71
+ event = events.last
72
+ Logger.info "Sifting event: #{event.inspect}"
73
+ ltime = event['LTime']
74
+ if last_worked.nil? || ltime > last_worked
75
+ yield event
76
+
77
+ Diplomat::Kv.put(KEY, ltime.to_s)
78
+ else
79
+ Logger.info 'Skipping duplicate event'
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,30 @@
1
+ require 'concurrent'
2
+ require 'concurrent-edge'
3
+ require 'consul_stockpile/watch_event'
4
+
5
+ module ConsulStockpile
6
+ class WatchEventActor < Concurrent::Actor::RestartingContext
7
+ def initialize(backup_actor:)
8
+ @backup_actor = backup_actor
9
+ tell :watch
10
+ end
11
+
12
+ def on_message(message)
13
+ if message == :watch
14
+ Logger.tagged('WatchEvent') do
15
+ begin
16
+ WatchEvent.call(
17
+ handler: ->(event){ @backup_actor << :backup }
18
+ )
19
+ rescue => e
20
+ Logger.warn "Warning: #{e.message}; retrying in 5 seconds"
21
+ Logger.warn e.backtrace
22
+ Concurrent::ScheduledTask.execute(5){ tell :watch }
23
+ end
24
+ end
25
+ else
26
+ pass
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,5 @@
1
+ require "consul_stockpile/version"
2
+
3
+ module ConsulStockpile
4
+ # Your code goes here...
5
+ end
metadata ADDED
@@ -0,0 +1,268 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: consul_stockpile
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Ryan Schlesinger
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2016-05-11 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: thor
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '0.19'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '0.19'
27
+ - !ruby/object:Gem::Dependency
28
+ name: concurrent-ruby
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: concurrent-ruby-edge
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '0.2'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '0.2'
55
+ - !ruby/object:Gem::Dependency
56
+ name: excon
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: 0.49.0
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: 0.49.0
69
+ - !ruby/object:Gem::Dependency
70
+ name: diplomat
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '0.17'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '0.17'
83
+ - !ruby/object:Gem::Dependency
84
+ name: fog-aws
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '0.9'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '0.9'
97
+ - !ruby/object:Gem::Dependency
98
+ name: mime-types
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '3.0'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '3.0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: bundler
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '1.11'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '1.11'
125
+ - !ruby/object:Gem::Dependency
126
+ name: rake
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '10.0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '10.0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: fog-local
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '0.3'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: '0.3'
153
+ - !ruby/object:Gem::Dependency
154
+ name: rspec
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: '3.4'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '3.4'
167
+ - !ruby/object:Gem::Dependency
168
+ name: fivemat
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '1.3'
174
+ type: :development
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '1.3'
181
+ - !ruby/object:Gem::Dependency
182
+ name: aruba
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '0.14'
188
+ type: :development
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '0.14'
195
+ - !ruby/object:Gem::Dependency
196
+ name: pry-byebug
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - ">="
200
+ - !ruby/object:Gem::Version
201
+ version: '0'
202
+ type: :development
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - ">="
207
+ - !ruby/object:Gem::Version
208
+ version: '0'
209
+ description:
210
+ email:
211
+ - ryan@outstand.com
212
+ executables:
213
+ - consul_stockpile
214
+ extensions: []
215
+ extra_rdoc_files: []
216
+ files:
217
+ - ".gitignore"
218
+ - Dockerfile
219
+ - Gemfile
220
+ - README.md
221
+ - Rakefile
222
+ - bin/console
223
+ - bin/setup
224
+ - consul_stockpile.gemspec
225
+ - docker-entrypoint.sh
226
+ - exe/consul_stockpile
227
+ - lib/consul_stockpile.rb
228
+ - lib/consul_stockpile/backup_consul_kv.rb
229
+ - lib/consul_stockpile/backup_consul_kv_actor.rb
230
+ - lib/consul_stockpile/base.rb
231
+ - lib/consul_stockpile/bootstrap_consul_kv.rb
232
+ - lib/consul_stockpile/bootstrap_consul_kv_actor.rb
233
+ - lib/consul_stockpile/cli.rb
234
+ - lib/consul_stockpile/consul_lock.rb
235
+ - lib/consul_stockpile/detect_canary.rb
236
+ - lib/consul_stockpile/detect_consul.rb
237
+ - lib/consul_stockpile/download_backup.rb
238
+ - lib/consul_stockpile/load_json_kv.rb
239
+ - lib/consul_stockpile/logger.rb
240
+ - lib/consul_stockpile/place_canary.rb
241
+ - lib/consul_stockpile/run_stockpile.rb
242
+ - lib/consul_stockpile/version.rb
243
+ - lib/consul_stockpile/watch_event.rb
244
+ - lib/consul_stockpile/watch_event_actor.rb
245
+ homepage: https://github.com/outstand/consul_stockpile
246
+ licenses: []
247
+ metadata: {}
248
+ post_install_message:
249
+ rdoc_options: []
250
+ require_paths:
251
+ - lib
252
+ required_ruby_version: !ruby/object:Gem::Requirement
253
+ requirements:
254
+ - - ">="
255
+ - !ruby/object:Gem::Version
256
+ version: '0'
257
+ required_rubygems_version: !ruby/object:Gem::Requirement
258
+ requirements:
259
+ - - ">="
260
+ - !ruby/object:Gem::Version
261
+ version: '0'
262
+ requirements: []
263
+ rubyforge_project:
264
+ rubygems_version: 2.6.3
265
+ signing_key:
266
+ specification_version: 4
267
+ summary: Bootstrap and backup for consul kv store
268
+ test_files: []