consul_watcher 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +19 -0
- data/README.md +4 -0
- data/Rakefile +22 -13
- data/VERSION +1 -1
- data/consul_watcher.gemspec +1 -0
- data/docs/docker-quickstart.md +9 -8
- data/exe/consul_watcher +2 -3
- data/lib/consul_watcher.rb +23 -13
- data/lib/consul_watcher/destination/amqp.rb +39 -8
- data/lib/consul_watcher/destination/jq.rb +25 -8
- data/lib/consul_watcher/diff.rb +0 -3
- data/lib/consul_watcher/filters.rb +45 -0
- data/lib/consul_watcher/rake_helper.rb +76 -0
- data/lib/consul_watcher/storage/consul.rb +56 -3
- data/lib/consul_watcher/watch_type/key.rb +61 -16
- metadata +19 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9e50318d48e6289402eb0458fe4e4223a548810de65f592400dc3e7012f4443b
|
4
|
+
data.tar.gz: 4e25cb74e2e614f9a874aa07a9e8627a2733233cc3503ebb61ea4bc94a09afcd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a5e77b1018fce3771ccc803f0341067761ea55bfb19330332f9152d9beef7ad47971ca3a28e61e56b787ee7b9b902bdb898177b08c598083efdbf3858fb5885c
|
7
|
+
data.tar.gz: 18fac72357f24ab0e88516c79439fcef5bab11cc1c7f766e03dc63ace7897ed8c30c673706c3cfcf2bc363c8bee9ced936255f200c339fb8e6a1aca671d716c7
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# Changelog
|
2
|
+
|
3
|
+
## 0.0.2
|
4
|
+
|
5
|
+
### New Functionality
|
6
|
+
- Adding loggers to classes
|
7
|
+
- Classes now use class helper to initialize variables
|
8
|
+
- Added rake tasks for standing up local testing cluster `rake up`
|
9
|
+
- Updated message formats. Messages now have `id` and `change_type` fields
|
10
|
+
- Added Consul backend storage for json diffs
|
11
|
+
|
12
|
+
### TODO for 0.1.0 release
|
13
|
+
- Finalize Key Watch functionality
|
14
|
+
- Finalize Consul Storage functionality
|
15
|
+
- Finalize AMQP Destination functionality
|
16
|
+
- Add rspec tests
|
17
|
+
- Rake tasks to manage releases
|
18
|
+
- Update documentation
|
19
|
+
|
data/README.md
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
# NOTICE
|
2
2
|
This gem is a work in progress. Please see the [TODO](https://github.com/fortman/consul_watcher/blob/master/docs/TODO.md) documentation
|
3
3
|
|
4
|
+
## First 0.0.1 release. Partial functionality
|
5
|
+
- [ruby gem](https://rubygems.org/gems/consul_watcher)
|
6
|
+
- [docker image](https://cloud.docker.com/repository/docker/rfortman/consul_watcher)
|
4
7
|
|
5
8
|
## Note on naming
|
6
9
|
This project name is a little confusing in that it called `consul_watcher`. It is wrapping the `consul watch` command line tool from Hashicorp. To avoid confusion, this project in the context of both the docker image and ruby gem will be referred to as `consul_watcher`. Any time that `consul watch` is used, that is a reference to the actualy Hashicorp command.
|
@@ -10,6 +13,7 @@ This git project produces a docker image. The basic premise of this docker imag
|
|
10
13
|
* parse the json output of the [consul watch](https://www.consul.io/docs/commands/watch.html) command
|
11
14
|
* compare the output to the previous consul watch output
|
12
15
|
* send the diff to a destination
|
16
|
+
|
13
17
|
The primary destination at this time is an AMQP topic exchange. The underlying logic in the docker image is a ruby gem. To make the gem modular and extensible, the functionality has been broken up into 3 sections; storage, watch_type, and destination.
|
14
18
|
|
15
19
|
If we want to do a diff of consul watches, we need to store the previous consul watch json. This is because consul watches kick off new executions on every change, so we can't store the previous run in memory. Storage is accomplished with [storage classes](https://github.com/fortman/consul_watcher/blob/master/docs/storage/storage.md). The purpose of the storage class is to just store and retrieve previous consul watch json. At this time, backend storage is planned for both local filesystem, and consul kv storage.
|
data/Rakefile
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require 'bundler/gem_tasks'
|
4
4
|
require 'open3'
|
5
|
+
require_relative 'lib/consul_watcher/rake_helper'
|
5
6
|
|
6
7
|
spec_file = Gem::Specification.load('consul_watcher.gemspec')
|
7
8
|
|
@@ -39,17 +40,25 @@ task docker_build: [:build] do
|
|
39
40
|
end
|
40
41
|
end
|
41
42
|
|
42
|
-
task :
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
43
|
+
task :start_deps do
|
44
|
+
cmd = 'docker-compose --file test/docker-compose.yml up -d consul rabbitmq'
|
45
|
+
ConsulWatcher::RakeHelper.exec(cmd)
|
46
|
+
urls = [
|
47
|
+
'http://localhost:8500/v1/status/leader',
|
48
|
+
'http://localhost:15672'
|
49
|
+
]
|
50
|
+
ConsulWatcher::RakeHelper.wait_for_urls(urls)
|
51
|
+
ConsulWatcher::RakeHelper.config_rabbitmq
|
52
|
+
end
|
53
|
+
|
54
|
+
task up: [:start_deps] do
|
55
|
+
cmd = 'docker-compose --file test/docker-compose.yml up -d consul-watcher'
|
56
|
+
_output, _status = ConsulWatcher::RakeHelper.exec(cmd)
|
57
|
+
puts 'Starting queue consumer'
|
58
|
+
ConsulWatcher::RakeHelper.consumer_start
|
59
|
+
end
|
60
|
+
|
61
|
+
task :down do
|
62
|
+
cmd = 'docker-compose --file test/docker-compose.yml down'
|
63
|
+
_output, _status = ConsulWatcher::RakeHelper.exec(cmd)
|
55
64
|
end
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.2
|
data/consul_watcher.gemspec
CHANGED
@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
|
|
22
22
|
# spec.add_development_dependency 'bundler', '~> 2.0'
|
23
23
|
spec.add_development_dependency 'rake', '~> 10.0'
|
24
24
|
spec.add_runtime_dependency 'bundler', '~> 2.0'
|
25
|
+
spec.add_runtime_dependency 'diplomat', '~> 2.2.4'
|
25
26
|
spec.add_dependency 'bunny', '~> 1.7.0'
|
26
27
|
spec.add_dependency 'hashdiff', '~> 0.3'
|
27
28
|
spec.add_dependency 'slop', '~> 4.6'
|
data/docs/docker-quickstart.md
CHANGED
@@ -2,14 +2,15 @@
|
|
2
2
|
|
3
3
|
The easiest way to get started with consul_watcher is with the docker image. The docker entry point will run a consul watch and then pipe the output to the ruby consul_watcher program. The behavior is mostly driven by passing environmental variables into docker. Environmental variables, command line parameters and a configuration file are being tested to determine which is best way to configure consul_watch. There is a docker compose file to test things out locally The following steps should have you working with a local cluster fairly quick. A rake task will be created to automate these steps at some point.
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
5
|
+
You can build and run the stack locally with the following steps:
|
6
|
+
|
7
|
+
From the root of the git repository execute these rake tasks<br/>
|
8
|
+
:> `rake`<br/>
|
9
|
+
:> `rake up`<br/>
|
10
|
+
|
11
|
+
You will have a locally running consul and rabbitmq instance. Rabbitmq will already have a queue bound to the appropriate exchange. The terminal where you ran `rake up` will show any key/value changes you make in consul.
|
12
|
+
|
13
|
+
After the docker containers have started, you can login to consul locally http://localhost:8500. You should be able to start creating, updating and deleting entries in the kv store. Go back to rabbitmq and you should be seeing messages flowing through the queue as you make updates in consul.<br/>
|
13
14
|
|
14
15
|
# environmental variables used by docker image
|
15
16
|
| environment variable | description | example value |
|
data/exe/consul_watcher
CHANGED
@@ -10,8 +10,6 @@ require 'consul_watcher'
|
|
10
10
|
logger = Logger.new(STDOUT)
|
11
11
|
|
12
12
|
opts = Slop.parse do |o|
|
13
|
-
o.string '--watch-type', required: true
|
14
|
-
o.string '--storage-name', required: true
|
15
13
|
o.string '--config-file', required: true
|
16
14
|
o.on '-h', '--help', 'print help' do
|
17
15
|
logger.warn("\n#{o}")
|
@@ -21,4 +19,5 @@ end
|
|
21
19
|
|
22
20
|
config = JSON.parse(File.read(opts['config-file']))
|
23
21
|
|
24
|
-
ConsulWatcher.watch(opts['watch-type'],
|
22
|
+
# ConsulWatcher.watch(opts['watch-type'], config)
|
23
|
+
ConsulWatcher.watch(config)
|
data/lib/consul_watcher.rb
CHANGED
@@ -4,38 +4,48 @@
|
|
4
4
|
|
5
5
|
require 'json'
|
6
6
|
require 'hashdiff'
|
7
|
+
require 'consul_watcher/filters'
|
7
8
|
|
8
9
|
# Top Level module to run watch logic
|
9
10
|
module ConsulWatcher
|
10
|
-
def self.watch(
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
destination = get_destination(config['destination']) if config['destination']
|
15
|
-
|
11
|
+
def self.watch(config)
|
12
|
+
#Encoding.default_external = Encoding::UTF_8
|
13
|
+
#Encoding.default_external = Encoding::ASCII_8BIT
|
14
|
+
assemble(config)
|
16
15
|
current_watch_json = $stdin.read
|
17
|
-
previous_watch_json = storage.fetch
|
18
|
-
changes = watch_type.get_changes(previous_watch_json, current_watch_json)
|
16
|
+
previous_watch_json = @storage.fetch
|
17
|
+
changes = @watch_type.get_changes(previous_watch_json, current_watch_json)
|
19
18
|
changes.each do |change|
|
20
|
-
destination.send(change)
|
19
|
+
@destination.send(change)
|
21
20
|
end
|
22
|
-
storage.push(
|
21
|
+
@storage.push(current_watch_json) unless changes.empty?
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def self.assemble(config)
|
27
|
+
@storage = get_storage(config['storage'])
|
28
|
+
@watch_type = get_watch_type(config['watch_type'])
|
29
|
+
@destination = get_destination(config['destination'])
|
30
|
+
|
31
|
+
@watch_type.filters = ConsulWatcher::Filters.new(config['watch_type'] || {})
|
32
|
+
@watch_type.filters.add_filters(@storage.get_filters)
|
23
33
|
end
|
24
34
|
|
25
35
|
def self.get_storage(storage_config)
|
26
|
-
classname = storage_config['classname']
|
36
|
+
classname = storage_config['classname']
|
27
37
|
require classname_to_file(classname)
|
28
38
|
Object.const_get(classname).new(storage_config)
|
29
39
|
end
|
30
40
|
|
31
41
|
def self.get_watch_type(watch_type_config)
|
32
|
-
classname = watch_type_config['classname']
|
42
|
+
classname = watch_type_config['classname']
|
33
43
|
require classname_to_file(classname)
|
34
44
|
Object.const_get(classname).new(watch_type_config)
|
35
45
|
end
|
36
46
|
|
37
47
|
def self.get_destination(destination_config)
|
38
|
-
classname = destination_config['classname']
|
48
|
+
classname = destination_config['classname']
|
39
49
|
require classname_to_file(classname)
|
40
50
|
Object.const_get(classname).new(destination_config)
|
41
51
|
end
|
@@ -1,28 +1,59 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'bunny'
|
4
|
+
require 'consul_watcher/class_helper'
|
5
|
+
require 'json'
|
4
6
|
|
5
7
|
module ConsulWatcher
|
6
8
|
module Destination
|
7
9
|
# Send diff output to jq command line
|
8
10
|
class Amqp
|
11
|
+
include ClassHelper
|
12
|
+
|
9
13
|
def initialize(destination_config)
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
14
|
+
populate_variables(destination_config)
|
15
|
+
setup_rabbitmq
|
16
|
+
end
|
17
|
+
|
18
|
+
def setup_rabbitmq
|
19
|
+
@conn = Bunny.new(host: @rabbitmq_server,
|
20
|
+
port: @rabbitmq_port,
|
21
|
+
vhost: @rabbitmq_vhost,
|
22
|
+
username: @rabbitmq_username,
|
23
|
+
password: @rabbitmq_password)
|
15
24
|
@conn.start
|
16
25
|
@ch = @conn.create_channel
|
17
26
|
@ex = Bunny::Exchange.new(@ch,
|
18
27
|
:topic,
|
19
|
-
|
28
|
+
@rabbitmq_exchange,
|
20
29
|
durable: true)
|
21
30
|
end
|
22
31
|
|
23
32
|
def send(change)
|
24
|
-
|
25
|
-
|
33
|
+
@logger.debug('publishing message')
|
34
|
+
routing_key = change['id']
|
35
|
+
@logger.debug("routing_key: #{routing_key}")
|
36
|
+
begin
|
37
|
+
@ex.publish(JSON.pretty_generate(change), routing_key: change['id'])
|
38
|
+
rescue Encoding::UndefinedConversionError
|
39
|
+
change['forced_utf8'] = true
|
40
|
+
data = change.to_s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
|
41
|
+
@ex.publish(JSON.pretty_generate(data), routing_key: change['id'])
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def defaults
|
46
|
+
logger = Logger.new(STDOUT)
|
47
|
+
logger.level = Logger::DEBUG
|
48
|
+
{
|
49
|
+
logger: logger,
|
50
|
+
rabbitmq_server: 'localhost',
|
51
|
+
rabbitmq_port: '5672',
|
52
|
+
rabbitmq_vhost: '/',
|
53
|
+
rabbitmq_username: 'guest',
|
54
|
+
rabbitmq_password: 'guest',
|
55
|
+
rabbitmq_exchange: 'amq.topic'
|
56
|
+
}
|
26
57
|
end
|
27
58
|
end
|
28
59
|
end
|
@@ -1,27 +1,44 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'open3'
|
4
|
+
require 'logger'
|
4
5
|
|
5
6
|
module ConsulWatcher
|
6
7
|
module Destination
|
7
8
|
# Send diff output to jq command line
|
8
9
|
class Jq
|
9
10
|
def initialize(destination_config)
|
11
|
+
populate_variables(destination_config)
|
10
12
|
end
|
11
13
|
|
12
14
|
def send(change)
|
13
15
|
change_json = JSON.pretty_generate(change)
|
14
16
|
Open3.popen3("/usr/bin/env jq '.'") do |stdin, stdout, stderr, wait_thr|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
17
|
+
{ out: stdout, err: stderr }.each do |_key, stream|
|
18
|
+
threads << Thread.new do
|
19
|
+
until (raw_line = stream.gets).nil?
|
20
|
+
output << raw_line.to_s
|
21
|
+
@logger.info(raw_line.to_s) if stream
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
threads.each(&:join)
|
26
|
+
status = wait_thr.value.success?
|
23
27
|
end
|
24
28
|
end
|
29
|
+
|
30
|
+
def defaults
|
31
|
+
logger = Logger.new(STDOUT)
|
32
|
+
logger.level = Logger::INFO
|
33
|
+
{
|
34
|
+
logger: logger,
|
35
|
+
rabbitmq_server: 'localhost',
|
36
|
+
rabbitmq_port: '5672',
|
37
|
+
rabbitmq_vhost: '/',
|
38
|
+
rabbitmq_username: 'guest',
|
39
|
+
rabbitmq_password: 'guest',
|
40
|
+
rabbitmq_exchange: 'amq.topic'
|
41
|
+
}
|
25
42
|
end
|
26
43
|
end
|
27
44
|
end
|
data/lib/consul_watcher/diff.rb
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'hashdiff'
|
4
|
-
require 'logger'
|
5
4
|
|
6
5
|
module ConsulWatcher
|
7
6
|
class Diff
|
@@ -10,10 +9,8 @@ module ConsulWatcher
|
|
10
9
|
def parse(previous_watch_data, current_watch_data)
|
11
10
|
diff = HashDiff.diff(sanitize_data(previous_watch_data), sanitize_data(current_watch_data), array_path: true)
|
12
11
|
diff.each do |change|
|
13
|
-
puts "change: #{change}"
|
14
12
|
change[1] = change[1]&.join('/')
|
15
13
|
end
|
16
|
-
puts "hashdiff: #{diff}"
|
17
14
|
diff
|
18
15
|
end
|
19
16
|
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'consul_watcher/class_helper'
|
4
|
+
|
5
|
+
module ConsulWatcher
|
6
|
+
class Filters
|
7
|
+
include ClassHelper
|
8
|
+
|
9
|
+
def initialize(filter_config)
|
10
|
+
populate_variables(filter_config)
|
11
|
+
end
|
12
|
+
|
13
|
+
def add_filters(filters_to_add)
|
14
|
+
@filters.merge!(filters_to_add)
|
15
|
+
end
|
16
|
+
|
17
|
+
def filter?(change)
|
18
|
+
@filters.each do |attribute, regex|
|
19
|
+
match = change.key?(attribute) ? change[attribute].match?(/#{regex}/) : false
|
20
|
+
if match
|
21
|
+
@logger.debug("filtered #{change['id']} #{attribute} on regex #{regex}")
|
22
|
+
return true
|
23
|
+
end
|
24
|
+
end
|
25
|
+
false
|
26
|
+
end
|
27
|
+
|
28
|
+
def print_filters
|
29
|
+
@filters.each do |filter|
|
30
|
+
@logger.debug("filter: #{filter}")
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def defaults
|
37
|
+
logger = Logger.new(STDOUT)
|
38
|
+
logger.level = Logger::WARN
|
39
|
+
{
|
40
|
+
logger: logger,
|
41
|
+
filters: {}
|
42
|
+
}
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# frozen_string_literal: true
|
4
|
+
|
5
|
+
require 'open3'
|
6
|
+
require 'net/http'
|
7
|
+
require 'bunny'
|
8
|
+
require 'json'
|
9
|
+
|
10
|
+
module ConsulWatcher
|
11
|
+
module RakeHelper
|
12
|
+
def self.exec(command, stream: true)
|
13
|
+
output = [] ; threads = [] ; status = nil
|
14
|
+
Open3.popen3(command) do |_stdin, stdout, stderr, wait_thr|
|
15
|
+
{ out: stdout, err: stderr }.each do |_key, stream|
|
16
|
+
threads << Thread.new do
|
17
|
+
until (raw_line = stream.gets).nil?
|
18
|
+
output << raw_line.to_s
|
19
|
+
puts raw_line.to_s if stream
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
threads.each(&:join)
|
24
|
+
status = wait_thr.value.success?
|
25
|
+
end
|
26
|
+
return output, status
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.wait_for_urls(urls)
|
30
|
+
urls.each do |url|
|
31
|
+
uri = URI(url)
|
32
|
+
error = true
|
33
|
+
Net::HTTP.start(uri.host, uri.port, read_timeout: 5, max_retries: 12) do |http|
|
34
|
+
while error
|
35
|
+
begin
|
36
|
+
response = http.request(Net::HTTP::Get.new(uri))
|
37
|
+
error = false
|
38
|
+
rescue EOFError
|
39
|
+
retry
|
40
|
+
end
|
41
|
+
end
|
42
|
+
raise Exception unless response.code == '200'
|
43
|
+
|
44
|
+
puts "up: #{url}"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.config_rabbitmq
|
50
|
+
conn = Bunny.new(host: 'localhost', port: '5672', vhost: '/', \
|
51
|
+
username: 'guest', password: 'guest')
|
52
|
+
conn.start
|
53
|
+
ch = conn.create_channel
|
54
|
+
ex = Bunny::Exchange.new(ch, :topic, 'amq.topic', durable: true)
|
55
|
+
@queue = ch.queue('consul_watcher', durable: true).bind(ex, routing_key: 'consul_watcher.key.#')
|
56
|
+
nil
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.process_message(delivery_info, properties, body)
|
60
|
+
puts "routing_key: #{delivery_info[:routing_key]}"
|
61
|
+
puts "properties: #{properties}"
|
62
|
+
puts "body:\n#{body}"
|
63
|
+
puts
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.consumer_start
|
67
|
+
@consumer ||= @queue.subscribe(block: true, &itself.method(:process_message))
|
68
|
+
nil
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.consumer_stop
|
72
|
+
@consumer&.cancel
|
73
|
+
@consumer = nil
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -1,13 +1,66 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'diplomat'
|
4
|
+
require 'consul_watcher/class_helper'
|
5
|
+
require 'zlib'
|
6
|
+
|
3
7
|
# This will be a module to store previous consul watch json to compare with previous watch data
|
4
8
|
module ConsulWatcher
|
5
9
|
module Storage
|
6
10
|
# Consul storage for previous watch data
|
7
|
-
class
|
11
|
+
class Consul
|
12
|
+
include ClassHelper
|
13
|
+
|
8
14
|
def initialize(storage_config)
|
9
|
-
|
15
|
+
populate_variables(storage_config)
|
16
|
+
config_diplomat
|
17
|
+
end
|
18
|
+
|
19
|
+
def fetch
|
20
|
+
@logger.debug('fetching state from consul')
|
21
|
+
data = Diplomat::Kv.get(cache_file_name)
|
22
|
+
data = Zlib::Inflate.inflate(data) if @encrypt
|
23
|
+
data
|
24
|
+
rescue Diplomat::KeyNotFound
|
25
|
+
'{}'
|
26
|
+
end
|
27
|
+
|
28
|
+
def push(data)
|
29
|
+
@logger.debug('pushing state to consul')
|
30
|
+
Diplomat::Kv.put(cache_file_name, @encrypt ? Zlib::Deflate.deflate(data, Zlib::BEST_COMPRESSION) : data)
|
31
|
+
end
|
32
|
+
|
33
|
+
def get_filters
|
34
|
+
{ 'key_path' => cache_file_name }
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def cache_file_name
|
40
|
+
"#{@parent_dir}/#{@storage_name}"
|
41
|
+
end
|
42
|
+
|
43
|
+
def config_diplomat
|
44
|
+
Diplomat.configure do |config|
|
45
|
+
# Set up a custom Consul URL
|
46
|
+
config.url = @consul_http_addr
|
47
|
+
# Set extra Faraday configuration options and custom access token (ACL)
|
48
|
+
config.options = {headers: {"X-Consul-Token" => @consul_token}}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def defaults
|
53
|
+
logger = Logger.new(STDOUT)
|
54
|
+
logger.level = Logger::WARN
|
55
|
+
{
|
56
|
+
logger: logger,
|
57
|
+
storage_name: 'kv',
|
58
|
+
parent_dir: 'watch/json-store',
|
59
|
+
consul_http_addr: 'http://localhost:8500',
|
60
|
+
consul_token: nil,
|
61
|
+
encrypt: false
|
62
|
+
}
|
10
63
|
end
|
11
64
|
end
|
12
65
|
end
|
13
|
-
end
|
66
|
+
end
|
@@ -1,31 +1,48 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'base64'
|
4
|
+
require 'consul_watcher/class_helper'
|
4
5
|
|
5
6
|
module ConsulWatcher
|
6
7
|
module WatchType
|
7
8
|
class Key
|
8
|
-
|
9
|
+
include ClassHelper
|
10
|
+
|
11
|
+
attr_accessor :filters
|
12
|
+
def initialize(watch_config)
|
13
|
+
populate_variables(watch_config)
|
14
|
+
end
|
9
15
|
|
10
16
|
def get_changes(previous_watch_json, current_watch_json)
|
11
|
-
json_diff =
|
17
|
+
json_diff = get_diff(previous_watch_json, current_watch_json)
|
18
|
+
|
19
|
+
changes = json_diff.each.collect do |change|
|
20
|
+
formatted_change = format_change('key', change)
|
21
|
+
decode(formatted_change) if @decode_values
|
22
|
+
formatted_change
|
23
|
+
end.compact
|
24
|
+
changes = changes.each.collect.reject {|change| @filters.filter?(change)}
|
25
|
+
changes
|
26
|
+
end
|
27
|
+
|
28
|
+
def filters=(f)
|
29
|
+
@filters = f
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
def get_diff(previous_watch_json, current_watch_json)
|
34
|
+
HashDiff.diff(json_to_hash(previous_watch_json),
|
12
35
|
json_to_hash(current_watch_json),
|
13
36
|
array_path: true)
|
14
|
-
json_diff.each.collect do |change|
|
15
|
-
# change[1] = change[1].join('/')
|
16
|
-
change[2] = Base64.decode64(change[2]) if change[2].is_a? String
|
17
|
-
change[3] = Base64.decode64(change[3]) if change[3].is_a? String
|
18
|
-
{
|
19
|
-
'watch_type' => 'key',
|
20
|
-
'id' => id(change),
|
21
|
-
'diff' => change
|
22
|
-
}
|
23
|
-
end
|
24
|
-
#json_diff.reject { |change| change[0] == '~' && change[1][-1] == 'ModifyIndex' }
|
25
37
|
end
|
26
38
|
|
27
|
-
def
|
28
|
-
|
39
|
+
def decode(change)
|
40
|
+
return unless change['element'] == 'Value'
|
41
|
+
|
42
|
+
change['old_value'] = Base64.decode64(change['old_value']) if change['old_value'].is_a? String
|
43
|
+
change['new_value'] = Base64.decode64(change['new_value']) if change['new_value'].is_a? String
|
44
|
+
change['new_value']['Value'] = Base64.decode64(change['new_value']['Value']) if change['change_type'] == '+' && change['new_value']['Value'].is_a?(String)
|
45
|
+
change['old_value']['Value'] = Base64.decode64(change['old_value']['Value']) if change['change_type'] == '-' && change['old_value']['Value'].is_a?(String)
|
29
46
|
end
|
30
47
|
|
31
48
|
def json_to_hash(json)
|
@@ -34,7 +51,35 @@ module ConsulWatcher
|
|
34
51
|
{ kv['Key'] => kv.reject { |key, _value| key == 'Key' } }
|
35
52
|
end.reduce({}, :merge)
|
36
53
|
end
|
37
|
-
|
54
|
+
|
55
|
+
def format_change(watch_type, change)
|
56
|
+
old_value, new_value = change_values(change)
|
57
|
+
{
|
58
|
+
'id' => "consul_watcher.key.#{change[1][0].tr('/', '.')}",
|
59
|
+
'watch_type' => watch_type,
|
60
|
+
'key_path' => change[1][0],
|
61
|
+
'key_property' => change[1][1],
|
62
|
+
'old_value' => old_value,
|
63
|
+
'new_value' => new_value
|
64
|
+
}
|
65
|
+
end
|
66
|
+
|
67
|
+
def routing_key(change)
|
68
|
+
end
|
69
|
+
|
70
|
+
def change_values(change)
|
71
|
+
# Return old_value, new_value
|
72
|
+
return nil, change[2] if change[0] == '+'
|
73
|
+
return change[2], nil if change[0] == '-'
|
74
|
+
return change[2], change[3] if change[0] == '~'
|
75
|
+
end
|
76
|
+
|
77
|
+
def defaults
|
78
|
+
{
|
79
|
+
watch_type: 'key',
|
80
|
+
decode_values: true
|
81
|
+
}
|
82
|
+
end
|
38
83
|
end
|
39
84
|
end
|
40
85
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: consul_watcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryan Fortman
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-05-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '2.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: diplomat
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 2.2.4
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 2.2.4
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: bunny
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -89,6 +103,7 @@ extensions: []
|
|
89
103
|
extra_rdoc_files: []
|
90
104
|
files:
|
91
105
|
- ".gitignore"
|
106
|
+
- CHANGELOG.md
|
92
107
|
- Dockerfile
|
93
108
|
- Gemfile
|
94
109
|
- README.md
|
@@ -118,6 +133,8 @@ files:
|
|
118
133
|
- lib/consul_watcher/destination/amqp.rb
|
119
134
|
- lib/consul_watcher/destination/jq.rb
|
120
135
|
- lib/consul_watcher/diff.rb
|
136
|
+
- lib/consul_watcher/filters.rb
|
137
|
+
- lib/consul_watcher/rake_helper.rb
|
121
138
|
- lib/consul_watcher/storage/consul.rb
|
122
139
|
- lib/consul_watcher/storage/disk.rb
|
123
140
|
- lib/consul_watcher/watch_type/checks.rb
|