aggkit 0.3.1.8768 → 0.3.4

data/bin/agggen

@@ -7,65 +7,65 @@ require 'optparse'
 UTIL = File.basename(__FILE__)
 
 @opts = {
-  token: ENV['GITLAB_API_PRIVATE_TOKEN'] || ENV['BOT_TOKEN'] || ENV['TOKEN'],
+  token:    ENV['GITLAB_API_PRIVATE_TOKEN'] || ENV['BOT_TOKEN'] || ENV['TOKEN'],
   endpoint: 'https://br.rnds.pro/api/v4',
-  exclude: 'core/mq,core/auth,core/ca,core/router,core/admin,core/filator',
-  gtmpl: '%name%:%id%',
-  ptmpl: '%badge% %name%:%id%',
+  exclude:  'core/mq,core/auth,core/ca,core/router,core/admin,core/filator',
+  gtmpl:    '%name%:%id%',
+  ptmpl:    '%badge% %name%:%id%'
 }
 
 
 @dynamic = {
-  badge: Proc.new do |object|
+  badge:         proc do |object|
     "#{object.web_url}/badges/master/pipeline.svg"
   end,
-  pipeline: Proc.new do |object|
+  pipeline:      proc do |object|
     "#{object.web_url}/pipelines"
   end,
-  redmine_badge:Proc.new do |object|
+  redmine_badge: proc do |object|
     "!#{object.web_url}/badges/master/pipeline.svg!:#{object.web_url}/pipelines"
   end,
-  redmine_link:Proc.new do |object|
+  redmine_link:  proc do |object|
     "\"#{object.name}\":#{object.web_url}"
   end,
 
-  gitlab_link:Proc.new do |object|
+  gitlab_link:   proc do |object|
     "[#{object.name}](#{object.web_url})"
   end,
-  gitlab_badge:Proc.new do |object|
+  gitlab_badge:  proc do |object|
     "[![#{object.name} status](#{object.web_url}/badges/master/pipeline.svg)](#{object.web_url})"
-  end,
+  end
 }
 
 @tempaltes = {
-  redmine: "|%redmine_badge%|%redmine_link%|%description%|",
+  redmine:  '|%redmine_badge%|%redmine_link%|%description%|',
   gredmine: "\n|*%name%*||%description%|",
-  gitlab: "* [ ] %gitlab_link% %description%",
-  ggitlab: "\n**%name%**",
+  gitlab:   '* [ ] %gitlab_link% %description%',
+  ggitlab:  "\n**%name%**"
 }
 
 parser = OptionParser.new do |o|
   o.banner = "Usage: #{UTIL} [options]"
 
-  o.on("--list", 'list projects') do
+  o.on('--list', 'list projects') do
     @opts[:list] = true
   end
 
-  o.on("--groups groups", 'process only these groups') do |groups|
+  o.on('--groups groups', 'process only these groups') do |groups|
     @opts[:groups] = groups.to_s.split(/[ ,;|]/).map(&:to_s).reject(&:empty?).uniq
   end
 
-  o.on("--gtmpl template", 'template to print group') do |tmpl|
+  o.on('--gtmpl template', 'template to print group') do |tmpl|
     tmpl = @tempaltes[tmpl.to_s.to_sym] || tmpl
     @opts[:gtmpl] = tmpl.to_s
   end
 
-  o.on("--ptmpl template", 'template to print projects') do |tmpl|
+  o.on('--ptmpl template', 'template to print projects') do |tmpl|
     tmpl = @tempaltes[tmpl.to_s.to_sym] || tmpl
     @opts[:ptmpl] = tmpl.to_s
   end
 
-  o.on("--exclude projects", 'exclude projects from selection') do |exclude|
+  o.on('--exclude projects', 'exclude projects from selection') do |exclude|
     @opts[:exclude] = exclude.to_s
   end
 
@@ -73,23 +73,22 @@ parser = OptionParser.new do |o|
     @opts[:endpoint] = endpoint.to_s.strip
   end
 
-  o.on("--token token", 'set api token') do |token|
+  o.on('--token token', 'set api token') do |token|
     @opts[:token] = token.to_s.strip
   end
-
 end
 parser.parse!
 
 @api = Gitlab.client(endpoint: @opts[:endpoint], private_token: @opts[:token])
 
-MAIN_GROUP='aggredator'
+MAIN_GROUP = 'aggredator'.freeze
 
 
 
-def each_group group
+def each_group(group)
   Enumerator.new do |y|
     y << group
-    
+
     @api.group_subgroups(group.id, per_page: 1000).each do |sub|
       each_group(sub).each do |g|
         y << g
@@ -98,7 +97,7 @@ def each_group group
   end
 end
 
-def each_project group
+def each_project(group)
   Enumerator.new do |y|
     @api.group_projects(group.id, per_page: 1000).each do |p|
       y << p
@@ -106,17 +105,16 @@ def each_project group
   end
 end
 
-def inspect_group group
+def inspect_group(group)
   result = {}
   result[:groups] = @api.group_subgroups(group.id).each_with_object({}) do |sub, ret|
     ret[sub.name] = inspect_group(sub)
   end
-  result[:projects]  = @api.group_projects(group.id).map(&:name)
+  result[:projects] = @api.group_projects(group.id).map(&:name)
   result
 end
 
-
-def render tmpl, substs, object
+def render(tmpl, substs, object)
   substs.reduce(tmpl) do |ret, s|
     if callable = @dynamic[s.to_sym]
       ret.gsub("%#{s}%", callable.call(object).to_s)
@@ -129,9 +127,9 @@ end
 
 
 if @opts[:list]
-  gsubsts = @opts[:gtmpl].scan(/%\w+%/).map{|pattern| pattern[/\w+/]}
-  psubsts = @opts[:ptmpl].scan(/%\w+%/).map{|pattern| pattern[/\w+/]}
-  
+  gsubsts = @opts[:gtmpl].scan(/%\w+%/).map{|pattern| pattern[/\w+/] }
+  psubsts = @opts[:ptmpl].scan(/%\w+%/).map{|pattern| pattern[/\w+/] }
+
 
   main = @api.groups(search: MAIN_GROUP).first
 
@@ -158,7 +156,7 @@ if @opts[:list]
       puts line
     end
   end
-  
+
   exit 0
 end
 
@@ -166,4 +164,3 @@ end
 STDERR.puts parser.help
 exit 1
 
-

data/bin/agglock

@@ -4,9 +4,9 @@ require 'optparse'
 require 'diplomat'
 
 @opts = {
-  url: 'http://localhost:8500',
+  url:     'http://localhost:8500',
   timeout: 10,
-  ttl: 30 * 60
+  ttl:     30 * 60
 }
 
 parser = OptionParser.new do |o|
@@ -48,9 +48,9 @@ end
 
 if resource = @opts[:lock]
   locker = {
-    Name: "#{resource}_locker_#{rand(999_999)}",
+    Name:     "#{resource}_locker_#{rand(999_999)}",
     Behavior: 'delete',
-    TTL: "#{@opts[:ttl]}s",
+    TTL:      "#{@opts[:ttl]}s",
     resource: resource
   }
   sessionid = Diplomat::Session.create(locker)
@@ -69,3 +69,4 @@ else
   STDERR.puts parser.help
   exit 1
 end
+

data/bin/aggmerge

@@ -40,7 +40,9 @@ def extend_hash(first, second)
     next unless second.key?(fk)
 
     sv = second[fk]
-    raise "Types of values not match(#{fv.class}, #{sv.class})" if fv.class != sv.class
+    if !fv.nil? && !sv.nil? && fv.class != sv.class
+      raise "Types of values not match(#{fv.class}, #{sv.class})" if fv.class != sv.class
+    end
 
     # Специальный случай потому что command не мерджится а заменяется
     if fk == 'command'
@@ -67,31 +69,32 @@ end
 def process_compose_hash(yml, dirname, parent = {})
   (yml['services'] || {}).each_pair do |name, service|
     next unless ext = service['extends']
+
     base = if ext.is_a? String
-             template = yml['services'][ext]
-             parent_service = (parent['services'] || {})[ext] || {}
-             extend_hash(parent_service.deep_dup, template)
-           elsif file = ext['file']
-             ENV.each_pair do |k, v|
-               file.gsub!("$#{k}", v)
-               file.gsub!("${#{k}}", v)
-             end
-
-             file_to_load = if File.exist?(dirname + '/' + file)
-                              dirname + '/' + file
-                            else
-                              file
-                    end
-
-             tmp = process_compose_hash(YAML.load(File.read(file_to_load)), File.dirname(file_to_load), service)
-
-             begin
-                 (tmp['services'][ext['service']] || {})
-               rescue StandardError
-                 {}
-               end
-           else
-             yml['services'][ext['service']]
+      template = yml['services'][ext]
+      parent_service = (parent['services'] || {})[ext] || {}
+      extend_hash(parent_service.deep_dup, template)
+    elsif file = ext['file']
+      ENV.each_pair do |k, v|
+        file.gsub!("$#{k}", v)
+        file.gsub!("${#{k}}", v)
+      end
+
+      file_to_load = if File.exist?(dirname + '/' + file)
+        dirname + '/' + file
+      else
+        file
+      end
+
+      tmp = process_compose_hash(YAML.load(File.read(file_to_load)), File.dirname(file_to_load), service)
+
+      begin
+          (tmp['services'][ext['service']] || {})
+        rescue StandardError
+          {}
+        end
+    else
+      yml['services'][ext['service']]
     end.deep_dup
 
     service.delete 'extends'
@@ -101,13 +104,13 @@ def process_compose_hash(yml, dirname, parent = {})
   yml
 end
 
-def remove_build_section hash
-  if hash.has_key?(:build) || hash.has_key?('build')
+def remove_build_section(hash)
+  if hash.key?(:build) || hash.key?('build')
     hash.delete(:build)
     hash.delete('build')
   end
 
-  hash.each_pair do |k, v|
+  hash.each_pair do |_k, v|
     if v.is_a?(Hash)
       remove_build_section(v)
     elsif v.is_a?(Array)
@@ -124,6 +127,7 @@ if File.basename($PROGRAM_NAME) == File.basename(__FILE__)
     if yml['version'] && parent['version'] && yml['version'] != parent['version']
       raise "version mismatch: #{file}"
     end
+
     ret = extend_hash(parent.deep_dup, yml)
     ret
   end
@@ -138,3 +142,4 @@ if File.basename($PROGRAM_NAME) == File.basename(__FILE__)
   end
 
 end
+

data/bin/aggstart

@@ -5,16 +5,24 @@ require 'aggkit'
 UTIL = File.basename(__FILE__)
 
 @opts = {
-  service: ENV['AGGREDATOR_SERVICE'],
-  consul: ENV['CONSUL_HOST'] || 'localhost',
+  service:    ENV['AGGREDATOR_SERVICE'],
+  consul:     ENV['CONSUL_HTTP_ADDR'] || ENV['CONSUL_HOST'] || 'localhost',
   consulconf: [],
-  depends: [],
+  depends:    []
 }
 
+@opts[:exec] = (begin
+                  idx = ARGV.index {|a| a.strip == '--' } + 1
+                  ARGV[idx..-1]
+                rescue StandardError
+                  []
+                end)
+
+
 parser = Aggkit::OptionParser.new do |o|
   o.banner = "Usage: #{UTIL} [options]"
 
-  o.on("--service <NAME>", 'Set service NAME. AGGREDATOR_SERVICE env can be used instead') do |service|
+  o.on('--service <NAME>', 'Set service NAME. AGGREDATOR_SERVICE env can be used instead') do |service|
     @opts[:service] = service.to_s
   end
 
@@ -22,7 +30,7 @@ parser = Aggkit::OptionParser.new do |o|
     @opts[:consul] = consul.to_s
   end
 
-  o.on("--consulconf=consul,consul/smev3...", 'Set list of folders with consul configs. Special: none - disable local consul') do |list|
+  o.on('--consulconf=consul,consul/smev3...', 'Set list of folders with consul configs. Special: none - disable local consul') do |list|
     @opts[:consulconf] = if list.to_s == 'none'
       nil
     else
@@ -30,28 +38,18 @@ parser = Aggkit::OptionParser.new do |o|
     end
   end
 
-  o.on("--depends=mq,db,router...", 'Set list of services to wait for ready before start') do |list|
+  o.on('--depends=mq,db,router...', 'Set list of services to wait for ready before start') do |list|
     @opts[:depends] = list.to_s.split(/[ ;,|]/).map(&:to_s).reject(&:empty?)
   end
-
-  o.on("--exec <FILE>", 'Set FILE to execute.') do |file|
-    @opts[:exec] = file.to_s
-  end
-
-
 end
 parser.parse!
 
 
 include Aggkit::Runner
 
-if @opts[:service].to_s.empty?
-  die "service name must be provided"
-end
+die 'service name must be provided' if @opts[:service].to_s.empty?
 
-if @opts[:exec].to_s.empty?
-  die "exec script must be provided"
-end
+die 'exec script must be provided' if @opts[:exec].empty?
 
 @opts[:node_name] = "#{@opts[:service]}_#{`hostname`.strip}".strip
 
@@ -64,7 +62,6 @@ init_service @opts[:service], @opts
 envsubst '/home/app/consul'
 
 Aggkit::Watcher.new.exec do |watcher|
-
   if @opts[:consulconf]
     configs = @opts[:consulconf].map do |folder|
       "-config-dir #{folder}"
@@ -76,9 +73,9 @@ Aggkit::Watcher.new.exec do |watcher|
   if File.exist?('/home/app/docker/start_checker.py')
     watcher.add '/home/app/docker/start_checker.py'
   end
-  
 
-  execute!("aggwait -t 20 --consul-addr=http://#{@opts[:consul]}:8500 --consul", "Timeout for consul service")  
+
+  execute!("aggwait -t 20 --consul-addr=http://#{@opts[:consul]}:8500 --consul", 'Timeout for consul service')
 
   load_envs_from_consul(@opts[:consul], @opts[:service])
 
@@ -88,6 +85,6 @@ Aggkit::Watcher.new.exec do |watcher|
 
   sleep 2
 
-  watcher.add %W{#{@opts[:exec]}}
+  watcher.add @opts[:exec]
 end
 

data/bin/aggterm

@@ -6,8 +6,8 @@ STDOUT.sync = true
 STDERR.sync = true
 
 @opts = {
-  code: 0,
-  sleep: 1,
+  code:      0,
+  sleep:     1,
   term_code: 0
 }
 
@@ -69,3 +69,4 @@ end
 
 log "normal exit with: #{@opts[:code]}"
 exit @opts[:code]
+

data/bin/aggwait

@@ -105,9 +105,9 @@ if @opts[:db]
   @pg[:db] = "-d #{@opts[:db]}"
   @pg[:user] = "-U #{@opts[:user]}" if @opts[:user]
   @pg[:pass] = if @opts[:pass] && !@opts[:pass].empty?
-                 "PGPASSWORD=#{@opts[:pass]}"
-               else
-                 ''
+    "PGPASSWORD=#{@opts[:pass]}"
+  else
+    ''
   end
 
   @pg[:host] = "-h #{@opts[:host]}" if @opts[:host]
@@ -124,6 +124,7 @@ def wait_for(timeout)
     return success if success
 
     return false if (Time.now - starttime) > timeout
+
     sleep @opts[:interval]
   end
 
@@ -280,3 +281,4 @@ if @opts[:docker]
     complete!(success)
   end
 end
+

data/bin/aggwrap

@@ -0,0 +1,250 @@
+#!/usr/bin/env ruby
+
+require 'diplomat'
+require 'securerandom'
+require 'aggkit'
+require 'net/http'
+
+STDERR.sync
+STDOUT.sync
+
+UTIL = File.basename(__FILE__)
+
+@opts = {
+  service: ENV['AGGREDATOR_SERVICE'],
+  id:      SecureRandom.hex(8),
+  consul:  ENV['CONSUL_HTTP_ADDR'] || ENV['CONSUL_HOST'] || 'localhost'
+}
+
+@opts[:exec] = (begin
+                  idx = ARGV.index {|a| a.strip == '--' } + 1
+                  ARGV[idx..-1]
+                rescue StandardError
+                  []
+                end)
+
+parser = Aggkit::OptionParser.new do |o|
+  o.banner = "Usage: #{UTIL} [options] -- exec script"
+
+  o.on('--service <NAME>', 'Set service NAME. AGGREDATOR_SERVICE env used when missed') do |service|
+    @opts[:service] = service.to_s
+  end
+
+  o.on('--id <identifier>', 'Set service identifier to concat with name. Random when default') do |id|
+    @opts[:id] = id.to_s
+  end
+
+  o.on("--consul=#{@opts[:consul]}", 'Set consul host. CONSUL_HOST env used when missed') do |consul|
+    @opts[:consul] = consul.to_s
+    Diplomat.configure do |config|
+      config.url = @opts[:consul]
+    end
+  end
+
+  o.on('--http <PORT>', 'Create TTL consul check and pass http result to it') do |port|
+    @opts[:http] = port.to_s
+  end
+
+  o.on('--file <FILE>', 'Create TTL consul check and pass file contents to it') do |file|
+    @opts[:file] = file.to_s
+  end
+
+  o.on('--pidfile <FILE>', 'Create TTL consul check and pass pid check(from file) to it') do |file|
+    @opts[:pidfile] = file.to_s
+  end
+
+  o.on('--script <FILE>', 'Create TTL consul check and pass script result to it') do |file|
+    @opts[:script] = file.to_s
+  end
+end
+parser.parse!
+
+die 'service name must be provided' if @opts[:service].to_s.empty?
+
+die 'script must be provided' if @opts[:exec].empty?
+
+SERVICE_ID = "#{@opts[:service]}_#{@opts[:id]}".freeze
+
+@service = {
+  Name:   @opts[:service],
+  ID:     SERVICE_ID,
+  Checks: []
+}
+
+@checks = {}
+
+
+@checks[:exec] = "#{SERVICE_ID}_exec_ttl"
+@service[:Checks].push(
+  CheckID:                        @checks[:exec],
+  Name:                           "Wrapped service: #{@opts[:service]}. Process: #{@opts[:exec].inspect}",
+  DeregisterCriticalServiceAfter: '1m',
+  TTL:                            '30s'
+)
+
+if @opts[:http]
+  @checks[:http] = "#{SERVICE_ID}_http_ttl"
+  @service[:Checks].push(
+    CheckID:                        @checks[:http],
+    Name:                           "Wrapped service: #{@opts[:service]}. HTTP: #{@opts[:http]}",
+    DeregisterCriticalServiceAfter: '1m',
+    TTL:                            '30s'
+  )
+end
+
+if @opts[:file]
+  @checks[:file] = "#{SERVICE_ID}_file_ttl"
+  @service[:Checks].push(
+    CheckID:                        @checks[:file],
+    Name:                           "Wrapped service: #{@opts[:service]}. File: #{@opts[:file]}",
+    DeregisterCriticalServiceAfter: '1m',
+    TTL:                            '30s'
+  )
+end
+
+if @opts[:pidfile]
+  @checks[:pidfile] = "#{SERVICE_ID}_pidfile_ttl"
+  @service[:Checks].push(
+    CheckID:                        @checks[:pidfile],
+    Name:                           "Wrapped service: #{@opts[:service]}. Pidfile: #{@opts[:pidfile]}",
+    DeregisterCriticalServiceAfter: '1m',
+    TTL:                            '30s'
+  )
+end
+
+if @opts[:script]
+  @checks[:script] = "#{SERVICE_ID}_script_ttl"
+  @service[:Checks].push(
+    CheckID:                        @checks[:script],
+    Name:                           "Wrapped service: #{@opts[:service]}. Script: #{@opts[:script]}",
+    DeregisterCriticalServiceAfter: '1m',
+    TTL:                            '30s'
+  )
+end
+
+def make_ttl_pass(check_id, output = nil)
+  STDERR.puts "TTL call failed: #{check_id}" unless Diplomat::Check.pass(check_id, output)
+end
+
+def make_ttl_fail(check_id, output = nil)
+  STDERR.puts "TTL call failed: #{check_id}" unless Diplomat::Check.fail(check_id, output)
+end
+
+def check_http(check_id, port)
+  response = Net::HTTP.get_response(URI("http://localhost:#{port}"))
+  output = response.body.strip
+  if response.code.to_i < 400
+    make_ttl_pass(check_id, output)
+  else
+    make_ttl_fail(check_id, output)
+  end
+rescue StandardError => e
+  make_ttl_fail(check_id, e.inspect)
+end
+
+def check_file(check_id, file)
+  output = File.read(file).strip
+  make_ttl_pass(check_id, output)
+rescue StandardError => e
+  make_ttl_fail(check_id, e.inspect)
+end
+
+def check_pidfile(check_id, pidfile)
+  pid = Integer(File.read(pidfile).strip)
+  make_ttl_pass(check_id)
+rescue StandardError => e
+  make_ttl_fail(check_id, e.inspect)
+end
+
+def check_script(check_id, script)
+  output = `#{script} 2>&1`.to_s.strip
+  if $?&.success?
+    make_ttl_pass(check_id, output)
+  else
+    make_ttl_fail(check_id, output)
+  end
+rescue StandardError => e
+  make_ttl_fail(check_id, e.inspect)
+end
+
+def run_check(check_id)
+  yield(check_id)
+rescue StandardError => e
+  STDERR.puts "make_ttl_pass[#{check_id}] Exception: #{e.inspect}"
+end
+
+if Diplomat::Service.register(@service)
+  Thread.new do
+    loop do
+      run_check(@checks[:exec]) do |check_id|
+        make_ttl_pass(check_id)
+      end
+
+      if @checks[:http]
+        run_check(@checks[:http]) do |check_id|
+          check_http(check_id, @opts[:http])
+        end
+      end
+
+      if @checks[:file]
+        run_check(@checks[:file]) do |check_id|
+          check_file(check_id, @opts[:file])
+        end
+      end
+
+      if @checks[:pidfile]
+        run_check(@checks[:pidfile]) do |check_id|
+          check_pidfile(check_id, @opts[:pidfile])
+        end
+      end
+
+      if @checks[:script]
+        run_check(@checks[:script]) do |check_id|
+          check_script(check_id, @opts[:script])
+        end
+      end
+
+      sleep 10
+    end
+  end
+else
+  die "can't register consul service"
+end
+
+$pid = fork do
+  exec(*@opts[:exec])
+end
+
+%w[INT TERM].each do |sig|
+  trap(sig) do
+    trap(sig, 'SYSTEM_DEFAULT')
+    ::Process.kill(sig, $pid)
+  end
+end
+
+pid, status = Process.wait2($pid)
+
+result = {}
+
+if status
+  if status.signaled?
+    result[:signal] = status.termsig
+  elsif status.exited?
+    result[:exit] = if status.success?
+      0
+    else
+      [status.exitstatus, 2].max
+    end
+  end
+end
+
+if result[:signal]
+  trap(result[:signal], 'SYSTEM_DEFAULT')
+  ::Process.kill(result[:signal], ::Process.pid)
+elsif result[:exit]
+  exit(result[:exit])
+else
+  STDERR.puts 'unknown subprocess result'
+  exit(-1)
+end
+