aggkit 0.2.5

data/bin/aggconsul

@@ -0,0 +1,222 @@
+#!/usr/bin/env ruby
+
+require 'diplomat'
+require 'optparse'
+require 'English'
+require 'yaml'
+
+STDOUT.sync = true
+STDERR.sync = true
+
+@opts = {}
+
+@opts[:exec] = (begin
+                  ARGV.join(' ').split(' -- ')[1].strip
+                rescue StandardError
+                  nil
+                end)
+
+parser = OptionParser.new do |o|
+  o.banner = 'Usage: consul.rb [options] -- exec'
+
+  o.on('--consul url', 'Set up a custom Consul URL') do |url|
+    Diplomat.configure do |config|
+      config.url = url.strip
+    end
+  end
+
+  o.on('--token token', 'Connect into consul with custom access token (ACL)') do |token|
+    Diplomat.configure do |config|
+      config.acl_token = token.strip
+    end
+  end
+
+  o.on('--init [service]', 'Initialize Consul services from config') do |service|
+    @opts[:service] = service
+    @opts[:init] = true
+  end
+
+  o.on('--config file', 'Read service configulation from file') do |file|
+    @opts[:config] = file.strip
+  end
+
+  o.on('--upload', 'Upload files to variables') do
+    @opts[:upload] = true
+  end
+
+  o.on('--show [service]', 'Show service configulation from Consul') do |service|
+    @opts[:service] = service
+    @opts[:show] = true
+  end
+
+  o.on('--override', 'override existed keys') do
+    @opts[:override] = true
+  end
+
+  o.on('-d', '--dereference', 'dereference consul values in form of "consul://key/subkey"') do
+    @opts[:dereference] = true
+  end
+
+  o.on('--env prefix', 'export KV values from prefix as env varaibles') do |prefix|
+    @opts[:env] = (prefix + '/').gsub('//', '/')
+  end
+
+  o.on('--export', 'add export to --env output') do
+    @opts[:export] = true
+  end
+
+  o.on('--pristine', "not include the parent processes' environment when exec child process") do
+    @opts[:pristine] = true
+  end
+
+  o.on('--put path:value', 'put value to path') do |path|
+    @opts[:put] = path.strip
+  end
+
+  o.on('--get path', 'get value from') do |path|
+    @opts[:get] = path.strip
+  end
+end
+parser.parse!
+
+def die(message)
+  STDERR.puts "Error: #{message}"
+  exit 1
+end
+
+def key_to_consul(key)
+  key.downcase.gsub(/[^0-9a-z]/i, '_')
+end
+
+def key_to_env(key)
+  key.upcase.gsub(/[^0-9a-z]/i, '_')
+end
+
+def dereferenced_value(value)
+  if @opts[:dereference] && value && value[/^consul:\/\//]
+    reference_path = value.gsub(/^consul:\/\//, '')
+    dereferenced_value(Diplomat::Kv.get(reference_path))
+  else
+    value
+  end
+end
+
+if config = @opts[:config]
+  @opts[:config] = YAML.load(config == '-' ? STDIN.read : File.read(config))
+end
+
+if @opts[:init]
+  raise OptionParser::MissingArgument.new('config') unless @opts[:config]
+
+  services = if service = @opts[:service]
+               {
+                 service => @opts[:config][service]
+               }
+             else
+               @opts[:config]
+  end
+
+  services.each_pair do |service, config|
+    next unless config
+    next if service[/^\./] # skip hidden keys
+
+    path = "services/env/#{service}"
+    config.each_pair do |env, item|
+      key = "#{path}/#{key_to_consul(env)}"
+      value = if @opts[:upload] && item['file']
+                File.read(item['file'])
+              else
+                item['value'] || item['default'] || item['file']
+      end
+
+      empty = begin
+        Diplomat::Kv.get(key)
+        false
+      rescue Diplomat::KeyNotFound
+        true
+      end
+
+      Diplomat::Kv.put(key, value.to_s.strip) || die("Can't put #{key} to Consul") if empty || @opts[:override]
+    end
+  end
+
+  exit 0
+end
+
+if @opts[:show]
+  config = {}
+
+  path = if service = @opts[:service]
+           "services/env/#{service}/"
+         else
+           'services/env/'
+         end
+
+  answer = Diplomat::Kv.get(path, recurse: true, convert_to_hash: true) || die("Can't get #{path} from Consul")
+  answer['services']['env'].each_pair do |service, env|
+    cfg = config[service] ||= {}
+
+    env.each_pair do |key, value|
+      value = dereferenced_value(value)
+
+      cfg[key_to_env(key)] = {
+        env: key_to_env(key),
+        value: value
+      }
+    end
+  end
+
+  STDOUT.puts JSON.parse(config.to_json).to_yaml
+
+  exit 0
+end
+
+if put = @opts[:put]
+  path, *value = put.split(':').map(&:strip)
+  value = value.join(':')
+  value = File.read(value) if @opts[:upload] && value && File.exist?(value)
+
+  Diplomat::Kv.put(path, value.to_s.strip) || die("Can't put #{path} to Consul")
+
+  exit 0
+end
+
+if path = @opts[:get]
+  value = dereferenced_value(Diplomat::Kv.get(path))
+
+  STDOUT.puts value.to_s.strip
+
+  exit 0
+end
+
+if prefix = @opts[:env]
+  keys = begin
+    Diplomat::Kv.get(prefix, keys: true)
+  rescue Diplomat::KeyNotFound => e
+    []
+  rescue StandardError
+    die("Can't get keys at #{prefix} from Consul")
+  end
+
+  env = keys.reduce({}) do |e, key|
+    value = dereferenced_value(Diplomat::Kv.get(key))
+
+    e.merge(key_to_env(key.gsub(prefix, '')) => value)
+  end
+
+  if cmd = @opts[:exec]
+    env = ENV.to_h.merge(env) unless @opts[:pristine]
+
+    exec(env, cmd, unsetenv_others: true)
+  else
+    env.each_pair do |k, v|
+      STDOUT.puts "#{@opts[:export] ? 'export ' : ''}#{k}=\"#{v}\""
+    end
+  end
+
+
+  exit 0
+end
+
+STDOUT.puts parser.help
+exit 1

data/bin/agglock

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'diplomat'
+
+@opts = {
+  url: 'http://localhost:8500',
+  timeout: 10,
+  ttl: 30 * 60
+}
+
+parser = OptionParser.new do |o|
+  o.banner = 'Usage: locker.rb [options]'
+
+  o.on("--consul url=#{@opts[:url]}", 'Set up a custom Consul URL') do |url|
+    Diplomat.configure do |config|
+      config.url = url.strip
+    end
+  end
+
+  o.on('--lock resource', 'resource name to lock with Consul') do |resource|
+    @opts[:lock] = resource.strip
+  end
+
+  o.on("--ttl seconds=#{@opts[:ttl]}", 'TTL to set when session created') do |seconds|
+    @opts[:ttl] = Integer(seconds.strip)
+  end
+
+  o.on('--unlock session', 'session name from previous call lock') do |session|
+    @opts[:unlock] = session.strip
+  end
+
+  o.on("--timeout seconds=#{@opts[:timeout]}", 'timeout to wait lock') do |seconds|
+    @opts[:timeout] = Integer(seconds.strip)
+  end
+end
+parser.parse!
+
+
+def lock(session, locker, timeout)
+  Timeout.timeout(timeout) do
+    return Diplomat::Lock.wait_to_acquire("resource/#{locker[:resource]}/lock", session, locker.to_json, 10)
+  end
+rescue Timeout::Error => e
+  false
+end
+
+
+if resource = @opts[:lock]
+  locker = {
+    Name: "#{resource}_locker_#{rand(999_999)}",
+    Behavior: 'delete',
+    TTL: "#{@opts[:ttl]}s",
+    resource: resource
+  }
+  sessionid = Diplomat::Session.create(locker)
+
+  if lock(sessionid, locker, @opts[:timeout])
+    puts sessionid
+    exit 0
+  else
+    STDERR.puts "Failed to lock resource: #{resource}"
+    exit 1
+  end
+
+elsif session = @opts[:unlock]
+  Diplomat::Session.destroy(session)
+else
+  STDERR.puts parser.help
+  exit 1
+end

data/bin/aggmerge

@@ -0,0 +1,118 @@
+#!/usr/bin/env ruby
+
+require 'English'
+require 'yaml'
+
+if File.basename($PROGRAM_NAME) == File.basename(__FILE__)
+  unless ENV['COMPOSE_FILE']
+    STDERR.puts 'COMPOSE_FILE environment must point to one or more files'
+    exit 1
+  end
+end
+
+
+class Hash
+
+  def deep_dup
+    Marshal.load(Marshal.dump(self))
+  end
+
+end
+
+class Array
+
+  def deep_dup
+    Marshal.load(Marshal.dump(self))
+  end
+
+end
+
+
+
+def extend_hash(first, second)
+  raise ArgumentError.new('First and second args equal nil') if [first, second].all? &:nil?
+  return second if first.nil?
+  return first if second.nil?
+
+  first.each_pair do |fk, fv|
+    next unless second.key?(fk)
+
+    sv = second[fk]
+    raise "Types of values not match(#{fv.class}, #{sv.class})" if fv.class != sv.class
+
+    # Специальный случай потому что command не мерджится а заменяется
+    if fk == 'command'
+      first[fk] = sv
+    elsif fv.is_a? Hash
+      extend_hash(fv, sv)
+    elsif fv.is_a? Array
+      fv |= sv
+      first[fk] = fv
+    else
+      first[fk] = sv
+    end
+  end
+
+  second.each_pair do |sk, sv|
+    next if first.key?(sk)
+
+    first[sk] = sv
+  end
+
+  first
+end
+
+def process_compose_hash(yml, dirname, parent = {})
+  (yml['services'] || {}).each_pair do |name, service|
+    next unless ext = service['extends']
+    base = if ext.is_a? String
+             template = yml['services'][ext]
+             parent_service = (parent['services'] || {})[ext] || {}
+             extend_hash(parent_service.deep_dup, template)
+           elsif file = ext['file']
+             ENV.each_pair do |k, v|
+               file.gsub!("$#{k}", v)
+               file.gsub!("${#{k}}", v)
+             end
+
+             file_to_load = if File.exist?(dirname + '/' + file)
+                              dirname + '/' + file
+                            else
+                              file
+                    end
+
+             tmp = process_compose_hash(YAML.load(File.read(file_to_load)), File.dirname(file_to_load), service)
+
+             begin
+                 (tmp['services'][ext['service']] || {})
+               rescue StandardError
+                 {}
+               end
+           else
+             yml['services'][ext['service']]
+    end.deep_dup
+
+    service.delete 'extends'
+
+    yml['services'][name] = extend_hash(base, service)
+  end
+  yml
+end
+
+if File.basename($PROGRAM_NAME) == File.basename(__FILE__)
+  result = ENV['COMPOSE_FILE'].split(':').reduce({}) do |parent, file|
+    yml = process_compose_hash(YAML.load(File.read(file)), File.dirname(file), parent)
+    if yml['version'] && parent['version'] && yml['version'] != parent['version']
+      raise "version mismatch: #{file}"
+    end
+    ret = extend_hash(parent.deep_dup, yml)
+    ret
+  end
+
+  if ARGV[0].nil? || ARGV[0].strip == '-'
+    puts YAML.dump(result)
+  else
+    File.write(ARGV[0].strip, YAML.dump(result))
+  end
+
+end

data/bin/aggwait

@@ -0,0 +1,262 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'securerandom'
+require 'English'
+require 'openssl'
+require 'tempfile'
+require 'json'
+
+STDOUT.sync = true
+STDERR.sync = true
+
+TIMEOUT = 15
+INTERVAL = 3
+
+@opts = {}
+
+@opts[:exec] = (begin
+                  ARGV.join(' ').split(' -- ')[1].strip
+                rescue StandardError
+                  nil
+                end)
+@opts[:timeout] = TIMEOUT
+@opts[:interval] = INTERVAL
+@opts[:consul_addr] = 'http://localhost:8500'
+@opts[:consul_service_count] = 1
+
+
+OptionParser.new do |o|
+  o.banner = 'Usage: waiter.rb [options] -- exec'
+
+  o.on('--tcp host:port', 'Wait for tcp accepts on host:port') do |addr|
+    host, port = addr.split(':')
+    @opts[:tcp] = addr.strip
+    @opts[:host] = host.strip
+    @opts[:port] = port.strip
+  end
+
+  o.on('--db dbname', 'Wait for PG database exists. Using --tcp to conenct PG') do |db|
+    @opts[:db] = db.strip
+  end
+
+  o.on('--tb tablename', 'Wait for PG table exists. Using --tcp to conenct PG') do |tb|
+    @opts[:tb] = tb.strip
+  end
+
+  o.on('-f', '--file filename', 'Wait for file exists.') do |file|
+    @opts[:file] = file.strip
+  end
+
+  o.on("--consul-addr addr=#{@opts[:consul_addr]}", 'HTTP addres to connect to consul') do |addr|
+    @opts[:consul_addr] = addr.strip
+  end
+
+  o.on('--consul', 'Wait for local consul agent to be ready') do
+    @opts[:consul] = true
+  end
+
+  o.on('--consul-service service', 'Wait for service appear in consul') do |service|
+    @opts[:consul_service] = service
+  end
+
+  o.on("--consul-service-count count=#{@opts[:consul_service_count]}", 'Wait for this count of service appear in consul') do |count|
+    @opts[:consul_service_count] = count.to_i
+  end
+
+  o.on('--consul-tag tag', 'Filter consul service by tag') do |tag|
+    @opts[:consul_tag] = tag.to_s
+  end
+
+  o.on('--user user', 'username') do |user|
+    @opts[:user] = user.strip
+  end
+
+  o.on('--pass pass', 'password') do |pass|
+    @opts[:pass] = pass.strip
+  end
+
+  o.on('-t', '--timeout secs=15', 'Total timeout') do |timeout|
+    @opts[:timeout] = timeout.to_i
+  end
+
+  o.on('-i', '--interval secs=2', 'Interval between attempts') do |interval|
+    @opts[:interval] = interval.to_i
+  end
+
+  o.on('-q', '--quiet', 'Do not output any status messages') do
+    @opts[:quiet] = true
+  end
+end.parse!
+
+def log(message)
+  puts message unless @opts[:quiet]
+end
+
+@opts[:timeout] = @opts[:timeout].to_i
+@opts[:interval] = @opts[:interval].to_i
+
+if @opts[:db]
+  @pg = {}
+  @pg[:db] = "-d #{@opts[:db]}"
+  @pg[:user] = "-U #{@opts[:user]}" if @opts[:user]
+  @pg[:pass] = if @opts[:pass] && !@opts[:pass].empty?
+                 "PGPASSWORD=#{@opts[:pass]}"
+               else
+                 ''
+  end
+
+  @pg[:host] = "-h #{@opts[:host]}" if @opts[:host]
+  @pg[:port] = "-p #{@opts[:port]}" if @opts[:port]
+
+  @pg[:tb] = @opts[:tb] if @opts[:tb]
+end
+
+def wait_for(timeout)
+  starttime = Time.now
+  loop do
+    success = yield
+
+    return success if success
+
+    return false if (Time.now - starttime) > timeout
+    sleep @opts[:interval]
+  end
+
+  false
+end
+
+def complete!(success)
+  if success
+    if @opts[:exec]
+      exec @opts[:exec]
+    else
+      exit 0
+    end
+  end
+
+  STDERR.puts 'Operation timed out'
+  exit 1
+end
+
+def wait_for_consul
+  log('Waiting for consul...')
+  ret = wait_for @opts[:timeout] do
+    cmd = "consul operator raft list-peers -http-addr=#{@opts[:consul_addr]} > /dev/null 2>&1"
+    system(cmd)
+    $?.success?
+  end
+
+  yield(ret)
+end
+
+def wait_for_consul_service(service)
+  log("Waiting for consul service #{service}...")
+  ret = wait_for @opts[:timeout] do
+    cmd = "curl -s #{@opts[:consul_addr]}/v1/health/service/#{service}?passing"
+    cmd += "&tag=#{@opts[:consul_tag]}" if @opts[:consul_tag]
+    result = `#{cmd}`
+    result = begin
+               JSON.parse(result)
+             rescue StandardError
+               []
+             end
+    result.count >= @opts[:consul_service_count]
+  end
+
+  yield(ret)
+end
+
+def wait_for_tcp
+  log("Waiting for TCP: #{@opts[:host]}:#{@opts[:port]}...")
+  ret = wait_for @opts[:timeout] do
+    cmd = "nc -z #{@opts[:host]} #{@opts[:port]} > /dev/null 2>&1"
+    system(cmd)
+    $?.success?
+  end
+
+  yield(ret)
+end
+
+def wait_for_db
+  log("Waiting for DB: pg://#{@opts[:user]}:#{@opts[:pass]}@#{@opts[:host]}:#{@opts[:port]}/#{@opts[:db]}...")
+  ret = wait_for @opts[:timeout] do
+    cmd = "#{@pg[:pass]} psql -lqt #{@pg[:user]} #{@pg[:host]} #{@pg[:port]} #{@pg[:db]} 2>/dev/null | cut -d \\| -f 1 | grep -qw #{@opts[:db]} > /dev/null 2>&1"
+    system(cmd)
+    $?.success?
+  end
+
+  yield(ret)
+end
+
+def wait_for_tb
+  log("Waiting for TABLE: pg://#{@opts[:user]}:#{@opts[:pass]}@#{@opts[:host]}:#{@opts[:port]}/#{@opts[:db]}##{@opts[:tb]}...")
+  ret = wait_for @opts[:timeout] do
+    cmd = "echo \"\\dt\" | psql -qt #{@pg[:user]} #{@pg[:pass]} #{@pg[:host]} #{@pg[:port]} #{@pg[:db]} 2>/dev/null | cut -d \\| -f 2 | grep -qw #{@pg[:tb]} > /dev/null 2>&1"
+    system(cmd)
+    $?.success?
+  end
+
+  yield(ret)
+end
+
+def wait_for_file(file = @opts[:file], timeout = @opts[:timeout])
+  log("Waiting for FILE: #{file}")
+  ret = wait_for timeout do
+    File.exist? file
+  end
+  yield(ret)
+end
+
+if @opts[:tb]
+  wait_for_tcp do |success|
+    if success
+      wait_for_db do |success|
+        if success
+          wait_for_tb do |success|
+            complete!(success)
+          end
+        end
+      end
+    end
+
+    complete!(false)
+  end
+end
+
+if @opts[:db]
+  wait_for_tcp do |success|
+    if success
+      wait_for_db do |success|
+        complete!(success)
+      end
+    end
+
+    complete!(false)
+  end
+end
+
+
+if @opts[:consul]
+  wait_for_consul do |success|
+    complete!(success)
+  end
+end
+
+if @opts[:consul_service]
+  wait_for_consul_service(@opts[:consul_service]) do |success|
+    complete!(success)
+  end
+end
+
+if @opts[:tcp]
+  wait_for_tcp do |success|
+    complete!(success)
+  end
+end
+
+if @opts[:file]
+  wait_for_file(@opts[:file], @opts[:timeout]) do |success|
+    complete!(success)
+  end
+end