consul_kv_backup 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 13b187d68d4ff6d1592949f06a38ff5cbb996005799ffb522759f66a4539d26e
+  data.tar.gz: aa9d14e725256b181d6d70c59718c4dda0c578f3e15faf1b17322ae395b7a89d
+SHA512:
+  metadata.gz: 239168aeaf537e277db14aa982bc4caf18fb07b95014c0001fd0fdec6ba725fc4951f3a48e1945fea3bffc9cba4e176f10160dafa50b2b9a208a1153aed3411c
+  data.tar.gz: 64d05b93d3d0fc99eb52f0c87f0d6b2c11179fe900f40bf4a995bf2b1d1c0e5fd1c479c32787d7376e91ae521ca8ef0aca5479ca7df1a5f410662eb2d7bb990d

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.vscode

data/Dockerfile

@@ -0,0 +1,13 @@
+FROM consul:1.4.0
+
+ARG gem_file
+
+RUN apk add --no-cache jq build-base libc-dev linux-headers postgresql-dev libxml2-dev libxslt-dev ruby ruby-dev ruby-rdoc ruby-irb git openssh-client
+
+COPY ./docker-entrypoint.sh /
+COPY pkg/$gem_file /consul_kv_backup/
+
+RUN cd /consul_kv_backup ; \
+    gem install $gem_file
+
+ENTRYPOINT ["/docker-entrypoint.sh"]

data/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+repo_name = '/fortman/consul_kv_backup'
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in consul_kv_backup.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Ryan Fortman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,2 @@
+# consul_kv_backup
+Backup changes in consul kv storge to a git repository.  Relies on the https://github.com/fortman/consul_watcher project.

data/Rakefile

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'open3'
+require 'flazm_ruby_helpers/os'
+require 'flazm_ruby_helpers/http'
+require 'flazm_ruby_helpers/project'
+
+spec_file = Gem::Specification.load('consul_kv_backup.gemspec')
+
+task default: :docker_build
+
+task :docker_tag, [:version, :docker_image_id] do |_task, args|
+  puts "Docker id #{args['docker_image_id']} => tag rfortman/consul_kv_backup:#{args['version']}"
+  tag_cmd = "docker tag #{args['docker_image_id']} rfortman/consul_kv_backup:#{args['version']}"
+  Open3.popen3(tag_cmd) do |_stdin, _stdout, stderr, wait_thr|
+    error = stderr.read
+    puts error unless wait_thr.value.success?
+  end
+end
+
+task docker_build: [:build] do
+  docker_image_id = nil
+  build_cmd = "docker build --build-arg gem_file=consul_kv_backup-#{spec_file.version}.gem ."
+  threads = []
+  Open3.popen3(build_cmd) do |_stdin, stdout, stderr, wait_thr|
+    { out: stdout, err: stderr }.each do |key, stream|
+      threads << Thread.new do
+        until (raw_line = stream.gets).nil?
+          match = raw_line.match(/Successfully built (.*)$/i)
+          docker_image_id = match.captures[0] if match
+          puts raw_line.to_s
+        end
+      end
+    end
+    threads.each(&:join)
+    if wait_thr.value.success?
+      Rake::Task['docker_tag'].invoke(spec_file.version, docker_image_id)
+      Rake::Task['docker_tag'].reenable
+      Rake::Task['docker_tag'].invoke('latest', docker_image_id)
+    end
+  end
+end
+
+task :start_deps do
+  cmd = 'docker-compose --file test/docker-compose.yml up -d consul rabbitmq'
+  FlazmRubyHelpers::Os.exec(cmd)
+  urls = [
+    'http://localhost:8500/v1/status/leader',
+    'http://localhost:15672'
+  ]
+  FlazmRubyHelpers::Http.wait_for_urls(urls)
+  cmd = 'docker-compose --file test/docker-compose.yml up -d consul-kv-watcher'
+  FlazmRubyHelpers::Os.exec(cmd)
+end
+
+task up: [:start_deps] do
+  cmd = 'docker-compose --file test/docker-compose.yml up consul-kv-backup'
+  _output, _status = FlazmRubyHelpers::Os.exec(cmd)
+end
+
+task :down do
+  cmd = 'docker-compose --file test/docker-compose.yml down'
+  _output, _status = FlazmRubyHelpers::Os.exec(cmd)
+end
+
+task publish: [:build, :docker_build] do
+  #FlazmRubyHelpers::Project::Git.publish(spec_file.version.to_s, 'origin', 'master')
+  FlazmRubyHelpers::Project::Docker.publish(spec_file.metadata['docker_image_name'], spec_file.version.to_s)
+  FlazmRubyHelpers::Project::Docker.publish(spec_file.metadata['docker_image_name'], 'latest')
+  FlazmRubyHelpers::Project::Gem.publish(spec_file.name.to_s, spec_file.version.to_s)
+end
+
+task :unpublish do
+  _output, _success = FlazmRubyHelpers::Os.exec("git tag --delete #{spec_file.version.to_s}")
+  _output, _success = FlazmRubyHelpers::Os.exec("gem yank #{spec_file.name.to_s} -v #{spec_file.version.to_s}")
+  puts "Please delete the tag from dockerhub at https://cloud.docker.com/repository/registry-1.docker.io/#{spec_file.metadata['docker_image_name']}/tags"
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/consul_kv_backup.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', '..')
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'consul_kv_backup'
+  spec.version       = IO.read('VERSION').chomp
+  spec.authors       = ['Ryan Fortman']
+  spec.email         = ['r.fortman.dev@gmail.com']
+
+  spec.summary       = 'Send consul watch events to an amqp.'
+  spec.homepage      = 'https://github.com/fortman/consul_kv_backup'
+  spec.metadata = {
+    'docker_image_name' => 'rfortman/consul_kv_backup'
+  } 
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  # spec.add_runtime_dependency 'bundler', '~> 2.0'
+  spec.add_runtime_dependency 'diplomat', '~> 2.2.4'
+  spec.add_dependency 'bunny', '~> 1.7.0'
+  spec.add_dependency 'slop', '~> 4.6'
+  spec.add_dependency 'flazm_ruby_helpers', '~> 0.0.3'
+end

data/docker-entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+echo
+echo "CLI_ARGS: \"${@}\""
+if [[ ${#} != 0 ]]; then
+  exec $@
+else
+  mkdir -p /repo ; cd /repo ; /usr/bin/consul_kv_backup --config-file /etc/consul_kv_backup/config.json
+  echo
+fi

data/exe/consul_kv_backup

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require 'slop'
+require 'logger'
+require 'json'
+require 'consul_kv_backup'
+
+logger = Logger.new(STDOUT)
+
+opts = Slop.parse do |o|
+  o.string '--config-file', required: true
+  o.on '-h', '--help', 'print help' do
+    logger.warn("\n#{o}")
+    exit(false)
+  end
+end
+
+config = JSON.parse(File.read(opts['config-file']))
+
+ConsulKvBackup.start(config)

data/lib/consul_kv_backup.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'consul_kv_backup/consul'
+require 'consul_kv_backup/amqp'
+require 'consul_kv_backup/git'
+
+module ConsulKvBackup
+  def self.start(config)
+    assemble(config)
+    @amqp.consul = @consul
+    @amqp.git = @git
+    @amqp.consume
+  end
+
+  def self.assemble(config)
+    @consul = ConsulKvBackup::Consul.new(config['consul'])
+    @amqp = ConsulKvBackup::Amqp.new(config['amqp'])
+    @git = ConsulKvBackup::Git.new(config['git'])
+  end
+end

data/lib/consul_kv_backup/amqp.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'bunny'
+require 'flazm_ruby_helpers/class'
+require 'json'
+
+module ConsulKvBackup
+  # Send diff output to jq command line
+  class Amqp
+    include FlazmRubyHelpers::Class
+
+    attr_accessor :consul, :git
+    def initialize(amqp_config)
+      initialize_variables(amqp_config)
+      setup_connection
+    end
+
+    def setup_connection
+      @conn = Bunny.new(amqp_opts)
+      @conn.start
+      @ch = @conn.create_channel
+      @ex = Bunny::Exchange.new(@ch,
+                                :topic,
+                                @amqp_exchange,
+                                durable: true)
+      @queue = @ch.queue(@amqp_queue, durable: true).bind(@ex, routing_key: @amqp_routing_key)
+    end
+
+    def consume
+      @consumer ||= @queue.subscribe(block: true, &itself.method(:process_message))
+      nil
+    end
+
+    private
+
+    def process_message(delivery_info, properties, body)
+      data = JSON.parse(body)
+      data['value'] = @consul.consul_key(data['key_path']) if data['new_value']
+      
+      @git.process_data(data)
+    end
+
+    def amqp_opts
+      opts = {}
+      opts[:vhost] = @amqp_vhost
+      opts[:username] = @amqp_username
+      opts[:password] = @amqp_password
+      if @amqp_addresses
+        opts[:addresses] = @amqp_addresses
+      elsif @amqp_hosts
+        opts[:hosts] = @amqp_hosts
+        opts[:port] = @amqp_port if @rabbitmq_port
+      elsif @amqp_host
+        opts[:host] = @amqp_host
+        opts[:port] = @amqp_port if @rabbitmq_port
+      end
+      opts
+    end
+
+    def defaults
+      logger = Logger.new(STDOUT)
+      logger.level = Logger::DEBUG
+      {
+        logger: logger,
+        amqp_host: 'localhost',
+        amqp_hosts: nil,
+        amqp_addresses: nil,
+        amqp_port: nil,
+        amqp_vhost: '/',
+        amqp_username: 'guest',
+        amqp_password: 'guest',
+        amqp_queue: nil,
+        amqp_exchange: "amq.topic",
+        amqp_routing_key: 'consul_watcher.key.#'
+      }
+    end
+  end
+end

data/lib/consul_kv_backup/consul.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'flazm_ruby_helpers/class'
+require 'diplomat'
+
+module ConsulKvBackup
+  # Consul storage for previous watch data
+  class Consul
+    include FlazmRubyHelpers::Class
+
+    def initialize(consul_config)
+      initialize_variables(consul_config)
+      setup_connection
+    end
+
+    def consul_key(key_path)
+      Diplomat::Kv.get(key_path)
+    end
+
+    private
+
+    def setup_connection
+      Diplomat.configure do |config|
+        # Set up a custom Consul URL
+        config.url = @consul_http_addr
+        # Set extra Faraday configuration options and custom access token (ACL)
+        config.options = {headers: {"X-Consul-Token" => @consul_token}}
+      end
+    end
+
+    def defaults
+      logger = Logger.new(STDOUT)
+      logger.level = Logger::WARN
+      {
+        logger: logger,
+        consul_http_addr: 'http://localhost:8500',
+        consul_token: nil,
+      }
+    end
+  end
+end

data/lib/consul_kv_backup/git.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'flazm_ruby_helpers/class'
+require 'flazm_ruby_helpers/os'
+require 'fileutils'
+
+module ConsulKvBackup
+  # Consul storage for previous watch data
+  class Git
+    include FlazmRubyHelpers::Class
+
+    def initialize(git_config)
+      initialize_variables(git_config)
+      git_setup
+    end
+
+    def process_data(data)
+      path = data['key_path']
+      value = data['value']
+      dc = data['consul_dc']
+
+      git_set_branch(dc)
+
+      if data['old_value'] and !data['new_value']
+        git_rm(path)
+      else
+        git_add(path, value)
+      end
+
+      # git_push if @git['push']
+    end
+
+    private
+
+    def git_setup
+      FileUtils.mkdir_p(@git_root_dir) unless File.directory?(@git_root_dir)
+      FlazmRubyHelpers::Os.exec("git config --global core.sshCommand '#{@git_config['core.sshCommand']}'")
+      unless File.directory?("#{@git_root_dir}/.git")
+        FlazmRubyHelpers::Os.exec("git clone #{@git_config['remote.origin.url']} .")
+      end
+      @git_config.each_pair do |key, value|
+        FlazmRubyHelpers::Os.exec("git config --local #{key} '#{value}'")
+      end
+      if FlazmRubyHelpers::Os.exec("git count-objects")[0][0].match(/0 objects, 0 kilobytes/)
+        FlazmRubyHelpers::Os.exec("git commit --allow-empty -m 'Initial commit'")
+      end
+    end
+
+    def git_set_branch(branch = 'master')
+      exist = git_branch_exists?(branch)
+      if exist
+        FlazmRubyHelpers::Os.exec("git reset --hard HEAD")
+        FlazmRubyHelpers::Os.exec("git checkout #{branch}")
+      else
+        FlazmRubyHelpers::Os.exec("git fetch")
+        FlazmRubyHelpers::Os.exec("git branch #{branch} master")
+        FlazmRubyHelpers::Os.exec("git checkout #{branch}")
+      end
+    end
+
+    def git_branch_exists?(branch)
+      FlazmRubyHelpers::Os.exec("git branch --list")[0].each do |line|
+        return true if line.match(/#{branch}/)
+      end
+      false
+    end
+
+    def git_add(path, value)
+      file = File.open(path, 'w')
+      file.write(value)
+      file.close
+      FlazmRubyHelpers::Os.exec('git add --all')
+      FlazmRubyHelpers::Os.exec('git commit -m "automated backup: updating"')
+    end
+
+    def git_rm(path)
+      FlazmRubyHelpers::Os.exec("git rm #{path}")
+      FlazmRubyHelpers::Os.exec('git commit -m "automated backup: deleting"')
+    end
+
+    def git_rebase
+      FlazmRubyHelpers::Os.exec('git fetch')
+      FlazmRubyHelpers::Os.exec('git reset origin/master')
+    end
+
+    def git_push
+      FlazmRubyHelpers::Os.exec('git push')
+    end
+
+    def defaults
+      {
+        git_root_dir: '/tmp/consul_backup',
+        git_config: {}
+      }
+    end
+  end
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: consul_kv_backup
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ryan Fortman
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-06-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: diplomat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.2.4
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.6'
+- !ruby/object:Gem::Dependency
+  name: flazm_ruby_helpers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+description: 
+email:
+- r.fortman.dev@gmail.com
+executables:
+- consul_kv_backup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Dockerfile
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- consul_kv_backup.gemspec
+- docker-entrypoint.sh
+- exe/consul_kv_backup
+- lib/consul_kv_backup.rb
+- lib/consul_kv_backup/amqp.rb
+- lib/consul_kv_backup/consul.rb
+- lib/consul_kv_backup/git.rb
+homepage: https://github.com/fortman/consul_kv_backup
+licenses: []
+metadata:
+  docker_image_name: rfortman/consul_kv_backup
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.8
+signing_key: 
+specification_version: 4
+summary: Send consul watch events to an amqp.
+test_files: []