consul_stockpile 0.1.2 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.dockerignore +3 -0
- data/Dockerfile +11 -10
- data/Dockerfile.release +7 -9
- data/README.md +6 -6
- data/build_dev.sh +39 -0
- data/consul_stockpile.gemspec +2 -1
- data/lib/consul_stockpile/backup_consul_kv.rb +20 -14
- data/lib/consul_stockpile/backup_consul_kv_actor.rb +3 -2
- data/lib/consul_stockpile/bootstrap_consul_kv.rb +18 -14
- data/lib/consul_stockpile/bootstrap_consul_kv_actor.rb +2 -1
- data/lib/consul_stockpile/bootstrap_external_services.rb +4 -2
- data/lib/consul_stockpile/cli.rb +3 -1
- data/lib/consul_stockpile/detect_canary.rb +5 -5
- data/lib/consul_stockpile/detect_consul.rb +6 -5
- data/lib/consul_stockpile/download_backup.rb +15 -14
- data/lib/consul_stockpile/load_json_kv.rb +9 -6
- data/lib/consul_stockpile/place_canary.rb +4 -2
- data/lib/consul_stockpile/run_stockpile.rb +25 -15
- data/lib/consul_stockpile/version.rb +1 -1
- data/lib/consul_stockpile/watch_event.rb +9 -7
- data/lib/consul_stockpile/watch_event_actor.rb +3 -2
- data/scripts/docker-entrypoint.sh +18 -0
- data/scripts/fetch-bundler-data.sh +16 -0
- metadata +31 -15
- data/docker-entrypoint.sh +0 -23
- data/lib/consul_stockpile/base.rb +0 -13
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 14f6e500b7641d1ef0812d1d2cb8d8f0f081c340
|
|
4
|
+
data.tar.gz: a1de2dffce6d3aa42374aacb71c3a68ebae9361a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e4ac5ee9e70bb0f8e3e6da6794b6544b463ad301e8bbf6deb6f904cd7bfcf35bc47910b45955663d3d885628c204e843dda6e034d53672389c65b57cb7aeefde
|
|
7
|
+
data.tar.gz: 51765f167f4e936b15db6be733372c04895440b8807de539d356bdbb313360b89d3fe84ddbe9bff9b4d12d9974f052c91d4c5c66605184e739ed1bc148c26d34
|
data/.dockerignore
ADDED
data/Dockerfile
CHANGED
|
@@ -1,26 +1,27 @@
|
|
|
1
|
-
FROM outstand/ruby-base:2.
|
|
1
|
+
FROM outstand/ruby-base:2.3.1-alpine
|
|
2
2
|
MAINTAINER Ryan Schlesinger <ryan@outstand.com>
|
|
3
3
|
|
|
4
4
|
RUN addgroup stockpile && \
|
|
5
5
|
adduser -S -G stockpile stockpile
|
|
6
6
|
|
|
7
|
-
RUN apk --no-cache add build-base
|
|
7
|
+
RUN apk --no-cache add build-base openssh
|
|
8
8
|
|
|
9
9
|
ENV USE_BUNDLE_EXEC true
|
|
10
10
|
|
|
11
|
+
WORKDIR /consul_stockpile
|
|
11
12
|
COPY Gemfile consul_stockpile.gemspec /consul_stockpile/
|
|
12
13
|
COPY lib/consul_stockpile/version.rb /consul_stockpile/lib/consul_stockpile/
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
COPY scripts/fetch-bundler-data.sh /consul_stockpile/scripts/fetch-bundler-data.sh
|
|
15
|
+
|
|
16
|
+
ARG bundler_data_host
|
|
17
|
+
RUN /consul_stockpile/scripts/fetch-bundler-data.sh ${bundler_data_host} && \
|
|
18
|
+
bundle install && \
|
|
19
|
+
git config --global push.default simple
|
|
17
20
|
COPY . /consul_stockpile/
|
|
18
21
|
RUN ln -s /consul_stockpile/exe/consul_stockpile /usr/local/bin/consul_stockpile
|
|
19
22
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
|
23
|
+
COPY scripts/docker-entrypoint.sh /docker-entrypoint.sh
|
|
23
24
|
|
|
24
25
|
ENV DUMB_INIT_SETSID 0
|
|
25
26
|
ENTRYPOINT ["/docker-entrypoint.sh"]
|
|
26
|
-
CMD ["
|
|
27
|
+
CMD ["help"]
|
data/Dockerfile.release
CHANGED
|
@@ -1,19 +1,17 @@
|
|
|
1
|
-
FROM outstand/ruby-base:2.
|
|
1
|
+
FROM outstand/ruby-base:2.3.1-alpine
|
|
2
2
|
MAINTAINER Ryan Schlesinger <ryan@outstand.com>
|
|
3
3
|
|
|
4
4
|
RUN addgroup stockpile && \
|
|
5
5
|
adduser -S -G stockpile stockpile
|
|
6
6
|
|
|
7
|
-
ENV CONSUL_STOCKPILE_VERSION=0.1.
|
|
7
|
+
ENV CONSUL_STOCKPILE_VERSION=0.1.5
|
|
8
8
|
|
|
9
|
-
RUN apk --no-cache add
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
&& gem install consul_stockpile -v ${CONSUL_STOCKPILE_VERSION} \
|
|
13
|
-
&& apk del build-dependencies
|
|
9
|
+
RUN apk --no-cache add --virtual build-dependencies build-base && \
|
|
10
|
+
gem install consul_stockpile -v ${CONSUL_STOCKPILE_VERSION} && \
|
|
11
|
+
apk del build-dependencies
|
|
14
12
|
|
|
15
|
-
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
|
13
|
+
COPY scripts/docker-entrypoint.sh /docker-entrypoint.sh
|
|
16
14
|
|
|
17
15
|
ENV DUMB_INIT_SETSID 0
|
|
18
16
|
ENTRYPOINT ["/docker-entrypoint.sh"]
|
|
19
|
-
CMD ["
|
|
17
|
+
CMD ["help"]
|
data/README.md
CHANGED
|
@@ -23,14 +23,14 @@ Or install it yourself as:
|
|
|
23
23
|
|
|
24
24
|
## Development
|
|
25
25
|
|
|
26
|
-
- `docker volume create --name
|
|
27
|
-
- `
|
|
28
|
-
- `docker run -it --rm --net=host -v $(pwd):/consul_stockpile -v
|
|
26
|
+
- `docker volume create --name consul_stockpile_fog`
|
|
27
|
+
- `./build_dev.sh`
|
|
28
|
+
- `docker run -it --rm --net=host -v $(pwd):/consul_stockpile -v consul_stockpile_fog:/fog -e FOG_LOCAL=true outstand/consul_stockpile:dev start -b bucket -n backup`
|
|
29
29
|
|
|
30
30
|
To release a new version:
|
|
31
|
-
- Update the version number in `version.rb` and commit the result.
|
|
32
|
-
- `
|
|
33
|
-
- `docker run -it --rm -v ~/.gitconfig:/
|
|
31
|
+
- Update the version number in `version.rb` and `Dockerfile.release` and commit the result.
|
|
32
|
+
- `./build_dev.sh`
|
|
33
|
+
- `docker run -it --rm -v ~/.gitconfig:/root/.gitconfig -v ~/.gitconfig.user:/root/.gitconfig.user -v ~/.ssh/id_rsa:/root/.ssh/id_rsa -v ~/.gem:/root/.gem outstand/consul_stockpile:dev rake release`
|
|
34
34
|
- `docker build -t outstand/consul_stockpile:VERSION -f Dockerfile.release .`
|
|
35
35
|
- `docker push outstand/consul_stockpile:VERSION`
|
|
36
36
|
|
data/build_dev.sh
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
set -e -x
|
|
3
|
+
|
|
4
|
+
build_image=outstand/consul_stockpile:dev
|
|
5
|
+
dockerfile=Dockerfile
|
|
6
|
+
bundler_data_dir=tmp
|
|
7
|
+
|
|
8
|
+
bundler_data_container=''
|
|
9
|
+
tar_container=''
|
|
10
|
+
|
|
11
|
+
function cleanup {
|
|
12
|
+
if [ -n "$bundler_data_container" ]; then
|
|
13
|
+
docker stop bundler-data
|
|
14
|
+
docker rm -fv bundler-data
|
|
15
|
+
fi
|
|
16
|
+
|
|
17
|
+
if [ -n "$tar_container" ]; then
|
|
18
|
+
docker rm -fv ${tar_container}
|
|
19
|
+
fi
|
|
20
|
+
|
|
21
|
+
rm -f ${bundler_data_dir}/cidfile
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
trap cleanup EXIT
|
|
25
|
+
|
|
26
|
+
build_args=''
|
|
27
|
+
mkdir -p ${bundler_data_dir}
|
|
28
|
+
|
|
29
|
+
if [ -f $(pwd)/${bundler_data_dir}/bundler-data.tar.gz ]; then
|
|
30
|
+
docker run --name bundler-data -v $(pwd)/${bundler_data_dir}/bundler-data.tar.gz:/usr/share/nginx/html/bundler-data.tar.gz:ro -d nginx:stable-alpine
|
|
31
|
+
bundler_data_container=bundler-data
|
|
32
|
+
build_args="--build-arg bundler_data_host=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' bundler-data)"
|
|
33
|
+
fi
|
|
34
|
+
|
|
35
|
+
docker build -t ${build_image} -f ${dockerfile} ${build_args} .
|
|
36
|
+
|
|
37
|
+
docker run -t --cidfile=${bundler_data_dir}/cidfile -w /usr/local/bundle ${build_image} tar -zcf /tmp/bundler-data.tar.gz .
|
|
38
|
+
tar_container=$(cat ${bundler_data_dir}/cidfile)
|
|
39
|
+
docker cp ${tar_container}:/tmp/bundler-data.tar.gz ${bundler_data_dir}/bundler-data.tar.gz
|
data/consul_stockpile.gemspec
CHANGED
|
@@ -24,10 +24,11 @@ Gem::Specification.new do |spec|
|
|
|
24
24
|
spec.add_runtime_dependency 'diplomat', '~> 0.17'
|
|
25
25
|
spec.add_runtime_dependency 'fog-aws', '~> 0.9'
|
|
26
26
|
spec.add_runtime_dependency 'mime-types', '~> 3.0'
|
|
27
|
+
spec.add_runtime_dependency 'metaractor', '~> 0.5'
|
|
28
|
+
spec.add_runtime_dependency "fog-local", "~> 0.3"
|
|
27
29
|
|
|
28
30
|
spec.add_development_dependency "bundler", "~> 1.11"
|
|
29
31
|
spec.add_development_dependency "rake", "~> 10.0"
|
|
30
|
-
spec.add_development_dependency "fog-local", "~> 0.3"
|
|
31
32
|
spec.add_development_dependency "rspec", "~> 3.4"
|
|
32
33
|
spec.add_development_dependency "fivemat", "~> 1.3"
|
|
33
34
|
spec.add_development_dependency "aruba", "~> 0.14"
|
|
@@ -1,18 +1,15 @@
|
|
|
1
|
-
require '
|
|
1
|
+
require 'metaractor'
|
|
2
2
|
require 'consul_stockpile/logger'
|
|
3
3
|
require 'fog/aws'
|
|
4
4
|
require 'fog/local' if ENV['FOG_LOCAL']
|
|
5
5
|
|
|
6
6
|
module ConsulStockpile
|
|
7
|
-
class BackupConsulKV
|
|
8
|
-
|
|
7
|
+
class BackupConsulKV
|
|
8
|
+
include Metaractor
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
EVENT_KEY = 'event/kv_update'.freeze
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
self.bucket = bucket
|
|
14
|
-
self.name = name
|
|
15
|
-
end
|
|
12
|
+
required :bucket, :name
|
|
16
13
|
|
|
17
14
|
def call
|
|
18
15
|
Logger.tagged('Backup') do
|
|
@@ -31,25 +28,34 @@ module ConsulStockpile
|
|
|
31
28
|
return
|
|
32
29
|
end
|
|
33
30
|
|
|
34
|
-
Logger.info "Uploading consul kv backup to bucket #{
|
|
31
|
+
Logger.info "Uploading consul kv backup to bucket #{bucket} as #{name}"
|
|
35
32
|
|
|
36
|
-
|
|
33
|
+
directory = nil
|
|
37
34
|
if ENV['FOG_LOCAL']
|
|
38
35
|
Logger.debug 'Using fog local storage'
|
|
39
36
|
storage = Fog::Storage.new provider: 'Local', local_root: '/fog'
|
|
40
|
-
|
|
37
|
+
directory = storage.directories.create(key: bucket)
|
|
41
38
|
else
|
|
42
39
|
storage = Fog::Storage.new provider: 'AWS', use_iam_profile: true
|
|
43
|
-
|
|
40
|
+
directory = storage.directories.get(bucket)
|
|
44
41
|
end
|
|
45
42
|
|
|
46
|
-
|
|
47
|
-
key: "#{
|
|
43
|
+
directory.files.create(
|
|
44
|
+
key: "#{name}.json",
|
|
48
45
|
body: data.to_json
|
|
49
46
|
)
|
|
50
47
|
|
|
51
48
|
Logger.info 'Backup complete.'
|
|
52
49
|
end
|
|
53
50
|
end
|
|
51
|
+
|
|
52
|
+
private
|
|
53
|
+
def bucket
|
|
54
|
+
context.bucket
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def name
|
|
58
|
+
context.name
|
|
59
|
+
end
|
|
54
60
|
end
|
|
55
61
|
end
|
|
@@ -13,13 +13,14 @@ module ConsulStockpile
|
|
|
13
13
|
if message == :backup
|
|
14
14
|
Logger.tagged('Backup') do
|
|
15
15
|
begin
|
|
16
|
-
BackupConsulKV.call(bucket: @bucket, name: @name)
|
|
16
|
+
BackupConsulKV.call!(bucket: @bucket, name: @name)
|
|
17
17
|
rescue => e
|
|
18
18
|
Logger.warn "Warning: #{e.message}; retrying in 5 seconds"
|
|
19
|
-
Logger.warn e.backtrace
|
|
19
|
+
Logger.warn e.backtrace.join("\n")
|
|
20
20
|
Concurrent::ScheduledTask.execute(5){ tell :backup }
|
|
21
21
|
end
|
|
22
22
|
end
|
|
23
|
+
nil
|
|
23
24
|
else
|
|
24
25
|
pass
|
|
25
26
|
end
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require '
|
|
1
|
+
require 'metaractor'
|
|
2
2
|
require 'consul_stockpile/consul_lock'
|
|
3
3
|
require 'consul_stockpile/detect_canary'
|
|
4
4
|
require 'consul_stockpile/download_backup'
|
|
@@ -8,34 +8,38 @@ require 'consul_stockpile/place_canary'
|
|
|
8
8
|
require 'consul_stockpile/bootstrap_external_services'
|
|
9
9
|
|
|
10
10
|
module ConsulStockpile
|
|
11
|
-
class BootstrapConsulKV
|
|
12
|
-
|
|
11
|
+
class BootstrapConsulKV
|
|
12
|
+
include Metaractor
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
LOCK_KEY = 'stockpile/bootstrap'.freeze
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
self.bucket = bucket
|
|
18
|
-
end
|
|
16
|
+
required :bucket
|
|
19
17
|
|
|
20
18
|
def call
|
|
21
19
|
Logger.tagged('Bootstrap') do
|
|
22
20
|
Logger.info 'Starting Consul KV Bootstrap...'
|
|
23
21
|
|
|
24
22
|
ConsulLock.with_lock(key: LOCK_KEY) do
|
|
25
|
-
if DetectCanary.call
|
|
23
|
+
if DetectCanary.call!.exists
|
|
26
24
|
Logger.info 'Canary detected; skipping bootstrap.'
|
|
27
|
-
|
|
25
|
+
context.ran_bootstrap = false
|
|
26
|
+
return
|
|
28
27
|
end
|
|
29
28
|
|
|
30
|
-
json = DownloadBackup.call(bucket: bucket).json_body
|
|
31
|
-
LoadJsonKV.call(json: json)
|
|
32
|
-
BootstrapExternalServices.call
|
|
33
|
-
PlaceCanary.call
|
|
29
|
+
json = DownloadBackup.call!(bucket: bucket).json_body
|
|
30
|
+
LoadJsonKV.call!(json: json)
|
|
31
|
+
BootstrapExternalServices.call!
|
|
32
|
+
PlaceCanary.call!
|
|
34
33
|
end
|
|
35
34
|
|
|
36
35
|
Logger.info 'Bootstrap complete.'
|
|
37
|
-
|
|
36
|
+
context.ran_bootstrap = true
|
|
38
37
|
end
|
|
39
38
|
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
def bucket
|
|
42
|
+
context.bucket
|
|
43
|
+
end
|
|
40
44
|
end
|
|
41
45
|
end
|
|
@@ -15,13 +15,14 @@ module ConsulStockpile
|
|
|
15
15
|
if message == :bootstrap
|
|
16
16
|
Logger.tagged('Bootstrap') do
|
|
17
17
|
begin
|
|
18
|
-
ran_bootstrap = BootstrapConsulKV.call(bucket: @bucket).ran_bootstrap
|
|
18
|
+
ran_bootstrap = BootstrapConsulKV.call!(bucket: @bucket).ran_bootstrap
|
|
19
19
|
@backup_actor << :backup if ran_bootstrap
|
|
20
20
|
rescue => e
|
|
21
21
|
Logger.warn "Warning: #{e.message}; retrying in 5 seconds"
|
|
22
22
|
Concurrent::ScheduledTask.execute(5){ tell :bootstrap }
|
|
23
23
|
end
|
|
24
24
|
end
|
|
25
|
+
nil
|
|
25
26
|
else
|
|
26
27
|
pass
|
|
27
28
|
end
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
require '
|
|
1
|
+
require 'metaractor'
|
|
2
2
|
require 'excon'
|
|
3
3
|
require 'diplomat'
|
|
4
4
|
require 'consul_stockpile/logger'
|
|
5
5
|
|
|
6
6
|
module ConsulStockpile
|
|
7
|
-
class BootstrapExternalServices
|
|
7
|
+
class BootstrapExternalServices
|
|
8
|
+
include Metaractor
|
|
9
|
+
|
|
8
10
|
KEY_PREFIX = 'stockpile/external_services'.freeze
|
|
9
11
|
URL = 'http://127.0.0.1:8500/v1/catalog/register'.freeze
|
|
10
12
|
|
data/lib/consul_stockpile/cli.rb
CHANGED
|
@@ -20,13 +20,15 @@ module ConsulStockpile
|
|
|
20
20
|
name: options[:name],
|
|
21
21
|
verbose: options[:verbose]
|
|
22
22
|
)
|
|
23
|
+
# actors are cleaned up in at_exit handler
|
|
24
|
+
exit 0
|
|
23
25
|
end
|
|
24
26
|
|
|
25
27
|
desc 'bootstrap_external_services', 'Copy external service entries from kv to catalog'
|
|
26
28
|
def bootstrap_external_services
|
|
27
29
|
$stdout.sync = true
|
|
28
30
|
require 'consul_stockpile/bootstrap_external_services'
|
|
29
|
-
BootstrapExternalServices.call
|
|
31
|
+
BootstrapExternalServices.call!
|
|
30
32
|
end
|
|
31
33
|
end
|
|
32
34
|
end
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
require '
|
|
1
|
+
require 'metaractor'
|
|
2
2
|
|
|
3
3
|
module ConsulStockpile
|
|
4
|
-
class DetectCanary
|
|
4
|
+
class DetectCanary
|
|
5
|
+
include Metaractor
|
|
6
|
+
|
|
5
7
|
KEY = 'stockpile/canary'.freeze
|
|
6
8
|
|
|
7
9
|
def call
|
|
8
10
|
canary = Diplomat::Kv.get(KEY, {}, :return)
|
|
9
|
-
|
|
10
|
-
exists: canary != ""
|
|
11
|
-
)
|
|
11
|
+
context.exists = canary != ''
|
|
12
12
|
end
|
|
13
13
|
end
|
|
14
14
|
end
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
require '
|
|
1
|
+
require 'metaractor'
|
|
2
2
|
require 'excon'
|
|
3
3
|
|
|
4
4
|
module ConsulStockpile
|
|
5
|
-
class DetectConsul
|
|
5
|
+
class DetectConsul
|
|
6
|
+
include Metaractor
|
|
7
|
+
|
|
6
8
|
URL = 'http://127.0.0.1:8500/v1/agent/self'.freeze
|
|
7
9
|
|
|
8
10
|
def call
|
|
@@ -19,9 +21,8 @@ module ConsulStockpile
|
|
|
19
21
|
rescue Excon::Errors::SocketError, Excon::Errors::HTTPStatusError
|
|
20
22
|
running = false
|
|
21
23
|
end
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
)
|
|
24
|
+
|
|
25
|
+
context.running = running
|
|
25
26
|
end
|
|
26
27
|
end
|
|
27
28
|
end
|
|
@@ -1,32 +1,30 @@
|
|
|
1
|
-
require '
|
|
1
|
+
require 'metaractor'
|
|
2
2
|
require 'consul_stockpile/logger'
|
|
3
3
|
require 'fog/aws'
|
|
4
4
|
require 'fog/local' if ENV['FOG_LOCAL']
|
|
5
5
|
|
|
6
6
|
module ConsulStockpile
|
|
7
|
-
class DownloadBackup
|
|
8
|
-
|
|
7
|
+
class DownloadBackup
|
|
8
|
+
include Metaractor
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
self.bucket = bucket
|
|
12
|
-
end
|
|
10
|
+
required :bucket
|
|
13
11
|
|
|
14
12
|
def call
|
|
15
|
-
Logger.info "Downloading consul kv backup from bucket #{
|
|
13
|
+
Logger.info "Downloading consul kv backup from bucket #{bucket}"
|
|
16
14
|
|
|
17
|
-
|
|
15
|
+
directory = nil
|
|
18
16
|
if ENV['FOG_LOCAL']
|
|
19
17
|
Logger.debug 'Using fog local storage'
|
|
20
18
|
storage = Fog::Storage.new provider: 'Local', local_root: '/fog'
|
|
21
|
-
|
|
19
|
+
directory = storage.directories.create(key: bucket)
|
|
22
20
|
else
|
|
23
21
|
storage = Fog::Storage.new provider: 'AWS', use_iam_profile: true
|
|
24
|
-
|
|
22
|
+
directory = storage.directories.get(bucket)
|
|
25
23
|
end
|
|
26
24
|
|
|
27
25
|
body = nil
|
|
28
26
|
last_modified = nil
|
|
29
|
-
|
|
27
|
+
directory.files.each do |file|
|
|
30
28
|
next if last_modified != nil && file.last_modified < last_modified
|
|
31
29
|
last_modified = file.last_modified
|
|
32
30
|
body = file.body
|
|
@@ -34,9 +32,12 @@ module ConsulStockpile
|
|
|
34
32
|
|
|
35
33
|
Logger.info 'Download complete.'
|
|
36
34
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
35
|
+
context.json_body = body
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
def bucket
|
|
40
|
+
context.bucket
|
|
40
41
|
end
|
|
41
42
|
end
|
|
42
43
|
end
|
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
require '
|
|
1
|
+
require 'metaractor'
|
|
2
2
|
require 'consul_stockpile/logger'
|
|
3
3
|
|
|
4
4
|
module ConsulStockpile
|
|
5
|
-
class LoadJsonKV
|
|
6
|
-
|
|
5
|
+
class LoadJsonKV
|
|
6
|
+
include Metaractor
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
self.json = json
|
|
10
|
-
end
|
|
8
|
+
optional :json
|
|
11
9
|
|
|
12
10
|
def call
|
|
13
11
|
Logger.info 'Loading json backup into consul kv store'
|
|
@@ -20,5 +18,10 @@ module ConsulStockpile
|
|
|
20
18
|
Diplomat::Kv.put(item['key'], item['value'])
|
|
21
19
|
end
|
|
22
20
|
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
def json
|
|
24
|
+
context.json
|
|
25
|
+
end
|
|
23
26
|
end
|
|
24
27
|
end
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require '
|
|
1
|
+
require 'metaractor'
|
|
2
2
|
require 'consul_stockpile/logger'
|
|
3
3
|
require 'concurrent'
|
|
4
4
|
require 'consul_stockpile/detect_consul'
|
|
@@ -7,17 +7,17 @@ require 'consul_stockpile/watch_event_actor'
|
|
|
7
7
|
require 'consul_stockpile/backup_consul_kv_actor'
|
|
8
8
|
|
|
9
9
|
module ConsulStockpile
|
|
10
|
-
class RunStockpile
|
|
11
|
-
|
|
10
|
+
class RunStockpile
|
|
11
|
+
include Metaractor
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
required :bucket, :name, :verbose
|
|
14
|
+
|
|
15
|
+
before do
|
|
16
|
+
context.verbose ||= false
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
def call
|
|
20
|
-
Concurrent.use_stdlib_logger(Logger::DEBUG) if
|
|
20
|
+
Concurrent.use_stdlib_logger(Logger::DEBUG) if verbose
|
|
21
21
|
|
|
22
22
|
self_read, self_write = IO.pipe
|
|
23
23
|
%w(INT TERM).each do |sig|
|
|
@@ -31,25 +31,24 @@ module ConsulStockpile
|
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
begin
|
|
34
|
-
while !DetectConsul.call
|
|
34
|
+
while !DetectConsul.call!.running
|
|
35
35
|
Logger.warn 'Local consul agent not detected, sleeping for 5 seconds'
|
|
36
36
|
sleep 5
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
backup_actor = BackupConsulKVActor.spawn(
|
|
40
40
|
:backup_consul_kv,
|
|
41
|
-
bucket:
|
|
42
|
-
name:
|
|
41
|
+
bucket: bucket,
|
|
42
|
+
name: name
|
|
43
43
|
)
|
|
44
44
|
|
|
45
45
|
BootstrapConsulKVActor.spawn(
|
|
46
46
|
:bootstrap_consul_kv,
|
|
47
47
|
backup_actor: backup_actor,
|
|
48
|
-
bucket:
|
|
48
|
+
bucket: bucket
|
|
49
49
|
)
|
|
50
50
|
WatchEventActor.spawn(:watch_event, backup_actor: backup_actor)
|
|
51
51
|
|
|
52
|
-
|
|
53
52
|
while readable_io = IO.select([self_read])
|
|
54
53
|
signal = readable_io.first[0].gets.strip
|
|
55
54
|
handle_signal(signal)
|
|
@@ -57,8 +56,6 @@ module ConsulStockpile
|
|
|
57
56
|
|
|
58
57
|
rescue Interrupt
|
|
59
58
|
Logger.info 'Exiting'
|
|
60
|
-
# actors are cleaned up in at_exit handler
|
|
61
|
-
exit 0
|
|
62
59
|
end
|
|
63
60
|
end
|
|
64
61
|
|
|
@@ -70,5 +67,18 @@ module ConsulStockpile
|
|
|
70
67
|
raise Interrupt
|
|
71
68
|
end
|
|
72
69
|
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
def bucket
|
|
73
|
+
context.bucket
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def name
|
|
77
|
+
context.name
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def verbose
|
|
81
|
+
context.verbose
|
|
82
|
+
end
|
|
73
83
|
end
|
|
74
84
|
end
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require '
|
|
1
|
+
require 'metaractor'
|
|
2
2
|
require 'excon'
|
|
3
3
|
require 'json'
|
|
4
4
|
require 'diplomat'
|
|
@@ -6,17 +6,15 @@ require 'consul_stockpile/consul_lock'
|
|
|
6
6
|
require 'consul_stockpile/logger'
|
|
7
7
|
|
|
8
8
|
module ConsulStockpile
|
|
9
|
-
class WatchEvent
|
|
9
|
+
class WatchEvent
|
|
10
|
+
include Metaractor
|
|
11
|
+
|
|
10
12
|
EVENT = 'kv_update'.freeze
|
|
11
13
|
URL = 'http://127.0.0.1:8500/v1/event/list'.freeze
|
|
12
14
|
KEY = 'event/kv_update'.freeze
|
|
13
15
|
LOCK_KEY = 'event/kv_update/lock'.freeze
|
|
14
16
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def initialize(handler:)
|
|
18
|
-
self.handler = handler
|
|
19
|
-
end
|
|
17
|
+
required :handler
|
|
20
18
|
|
|
21
19
|
def call
|
|
22
20
|
Logger.tagged('WatchEvent') do
|
|
@@ -52,6 +50,10 @@ module ConsulStockpile
|
|
|
52
50
|
end
|
|
53
51
|
|
|
54
52
|
private
|
|
53
|
+
def handler
|
|
54
|
+
context.handler
|
|
55
|
+
end
|
|
56
|
+
|
|
55
57
|
def handle_events(events)
|
|
56
58
|
events = JSON.parse(events)
|
|
57
59
|
return if events.empty?
|
|
@@ -13,15 +13,16 @@ module ConsulStockpile
|
|
|
13
13
|
if message == :watch
|
|
14
14
|
Logger.tagged('WatchEvent') do
|
|
15
15
|
begin
|
|
16
|
-
WatchEvent.call(
|
|
16
|
+
WatchEvent.call!(
|
|
17
17
|
handler: ->(event){ @backup_actor << :backup }
|
|
18
18
|
)
|
|
19
19
|
rescue => e
|
|
20
20
|
Logger.warn "Warning: #{e.message}; retrying in 5 seconds"
|
|
21
|
-
Logger.warn e.backtrace
|
|
21
|
+
Logger.warn e.backtrace.join("\n")
|
|
22
22
|
Concurrent::ScheduledTask.execute(5){ tell :watch }
|
|
23
23
|
end
|
|
24
24
|
end
|
|
25
|
+
nil
|
|
25
26
|
else
|
|
26
27
|
pass
|
|
27
28
|
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
#!/bin/dumb-init /bin/sh
|
|
2
|
+
set -e
|
|
3
|
+
|
|
4
|
+
if [ -n "$USE_BUNDLE_EXEC" ]; then
|
|
5
|
+
BINARY="bundle exec consul_stockpile"
|
|
6
|
+
else
|
|
7
|
+
BINARY=consul_stockpile
|
|
8
|
+
fi
|
|
9
|
+
|
|
10
|
+
if ${BINARY} help "$1" 2>&1 | grep -q "consul_stockpile $1"; then
|
|
11
|
+
set -- gosu stockpile ${BINARY} "$@"
|
|
12
|
+
|
|
13
|
+
if [ -n "$FOG_LOCAL" ]; then
|
|
14
|
+
chown -R stockpile:stockpile /fog
|
|
15
|
+
fi
|
|
16
|
+
fi
|
|
17
|
+
|
|
18
|
+
exec "$@"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
set -x
|
|
3
|
+
|
|
4
|
+
if [ -z "$1" ]; then
|
|
5
|
+
echo 'Missing argument: bundler data host'
|
|
6
|
+
exit 0
|
|
7
|
+
fi
|
|
8
|
+
|
|
9
|
+
wget -T 2 -O /tmp/bundler-data.tar.gz http://${1}/bundler-data.tar.gz
|
|
10
|
+
if [ $? -ne 0 ]; then
|
|
11
|
+
echo 'Unable to fetch bundler data.'
|
|
12
|
+
else
|
|
13
|
+
cd /usr/local/bundle && \
|
|
14
|
+
tar -zxf /tmp/bundler-data.tar.gz && \
|
|
15
|
+
rm /tmp/bundler-data.tar.gz
|
|
16
|
+
fi
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: consul_stockpile
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ryan Schlesinger
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-
|
|
11
|
+
date: 2016-07-18 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: thor
|
|
@@ -109,47 +109,61 @@ dependencies:
|
|
|
109
109
|
- !ruby/object:Gem::Version
|
|
110
110
|
version: '3.0'
|
|
111
111
|
- !ruby/object:Gem::Dependency
|
|
112
|
-
name:
|
|
112
|
+
name: metaractor
|
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
|
114
114
|
requirements:
|
|
115
115
|
- - "~>"
|
|
116
116
|
- !ruby/object:Gem::Version
|
|
117
|
-
version: '
|
|
118
|
-
type: :
|
|
117
|
+
version: '0.5'
|
|
118
|
+
type: :runtime
|
|
119
119
|
prerelease: false
|
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
|
121
121
|
requirements:
|
|
122
122
|
- - "~>"
|
|
123
123
|
- !ruby/object:Gem::Version
|
|
124
|
-
version: '
|
|
124
|
+
version: '0.5'
|
|
125
125
|
- !ruby/object:Gem::Dependency
|
|
126
|
-
name:
|
|
126
|
+
name: fog-local
|
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
|
128
128
|
requirements:
|
|
129
129
|
- - "~>"
|
|
130
130
|
- !ruby/object:Gem::Version
|
|
131
|
-
version: '
|
|
131
|
+
version: '0.3'
|
|
132
|
+
type: :runtime
|
|
133
|
+
prerelease: false
|
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
135
|
+
requirements:
|
|
136
|
+
- - "~>"
|
|
137
|
+
- !ruby/object:Gem::Version
|
|
138
|
+
version: '0.3'
|
|
139
|
+
- !ruby/object:Gem::Dependency
|
|
140
|
+
name: bundler
|
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
|
142
|
+
requirements:
|
|
143
|
+
- - "~>"
|
|
144
|
+
- !ruby/object:Gem::Version
|
|
145
|
+
version: '1.11'
|
|
132
146
|
type: :development
|
|
133
147
|
prerelease: false
|
|
134
148
|
version_requirements: !ruby/object:Gem::Requirement
|
|
135
149
|
requirements:
|
|
136
150
|
- - "~>"
|
|
137
151
|
- !ruby/object:Gem::Version
|
|
138
|
-
version: '
|
|
152
|
+
version: '1.11'
|
|
139
153
|
- !ruby/object:Gem::Dependency
|
|
140
|
-
name:
|
|
154
|
+
name: rake
|
|
141
155
|
requirement: !ruby/object:Gem::Requirement
|
|
142
156
|
requirements:
|
|
143
157
|
- - "~>"
|
|
144
158
|
- !ruby/object:Gem::Version
|
|
145
|
-
version: '0
|
|
159
|
+
version: '10.0'
|
|
146
160
|
type: :development
|
|
147
161
|
prerelease: false
|
|
148
162
|
version_requirements: !ruby/object:Gem::Requirement
|
|
149
163
|
requirements:
|
|
150
164
|
- - "~>"
|
|
151
165
|
- !ruby/object:Gem::Version
|
|
152
|
-
version: '0
|
|
166
|
+
version: '10.0'
|
|
153
167
|
- !ruby/object:Gem::Dependency
|
|
154
168
|
name: rspec
|
|
155
169
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -214,6 +228,7 @@ executables:
|
|
|
214
228
|
extensions: []
|
|
215
229
|
extra_rdoc_files: []
|
|
216
230
|
files:
|
|
231
|
+
- ".dockerignore"
|
|
217
232
|
- ".gitignore"
|
|
218
233
|
- Dockerfile
|
|
219
234
|
- Dockerfile.release
|
|
@@ -223,13 +238,12 @@ files:
|
|
|
223
238
|
- Rakefile
|
|
224
239
|
- bin/console
|
|
225
240
|
- bin/setup
|
|
241
|
+
- build_dev.sh
|
|
226
242
|
- consul_stockpile.gemspec
|
|
227
|
-
- docker-entrypoint.sh
|
|
228
243
|
- exe/consul_stockpile
|
|
229
244
|
- lib/consul_stockpile.rb
|
|
230
245
|
- lib/consul_stockpile/backup_consul_kv.rb
|
|
231
246
|
- lib/consul_stockpile/backup_consul_kv_actor.rb
|
|
232
|
-
- lib/consul_stockpile/base.rb
|
|
233
247
|
- lib/consul_stockpile/bootstrap_consul_kv.rb
|
|
234
248
|
- lib/consul_stockpile/bootstrap_consul_kv_actor.rb
|
|
235
249
|
- lib/consul_stockpile/bootstrap_external_services.rb
|
|
@@ -245,6 +259,8 @@ files:
|
|
|
245
259
|
- lib/consul_stockpile/version.rb
|
|
246
260
|
- lib/consul_stockpile/watch_event.rb
|
|
247
261
|
- lib/consul_stockpile/watch_event_actor.rb
|
|
262
|
+
- scripts/docker-entrypoint.sh
|
|
263
|
+
- scripts/fetch-bundler-data.sh
|
|
248
264
|
homepage: https://github.com/outstand/consul_stockpile
|
|
249
265
|
licenses: []
|
|
250
266
|
metadata: {}
|
|
@@ -264,7 +280,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
264
280
|
version: '0'
|
|
265
281
|
requirements: []
|
|
266
282
|
rubyforge_project:
|
|
267
|
-
rubygems_version: 2.6.
|
|
283
|
+
rubygems_version: 2.6.6
|
|
268
284
|
signing_key:
|
|
269
285
|
specification_version: 4
|
|
270
286
|
summary: Bootstrap and backup for consul kv store
|
data/docker-entrypoint.sh
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
#!/bin/dumb-init /bin/sh
|
|
2
|
-
set -e
|
|
3
|
-
|
|
4
|
-
if [ -n "$USE_BUNDLE_EXEC" ]; then
|
|
5
|
-
STOCKPILE_BINARY="bundle exec consul_stockpile"
|
|
6
|
-
else
|
|
7
|
-
STOCKPILE_BINARY=consul_stockpile
|
|
8
|
-
fi
|
|
9
|
-
|
|
10
|
-
if ${STOCKPILE_BINARY} help "$1" 2>&1 | grep -q "consul_stockpile $1"; then
|
|
11
|
-
set -- ${STOCKPILE_BINARY} "$@"
|
|
12
|
-
fi
|
|
13
|
-
|
|
14
|
-
if [ "$1" = "${STOCKPILE_BINARY}" ]; then
|
|
15
|
-
# Enable this to run as an unprivileged user
|
|
16
|
-
set -- gosu stockpile "$@"
|
|
17
|
-
|
|
18
|
-
if [ -n "$FOG_LOCAL" ]; then
|
|
19
|
-
chown -R stockpile:stockpile /fog
|
|
20
|
-
fi
|
|
21
|
-
fi
|
|
22
|
-
|
|
23
|
-
exec "$@"
|