port-authority 0.4.8 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c6efb1e9b1f802cf04ea83d0ab8703cd15cad51e
-  data.tar.gz: bdf0153379d87f1cbc1f02e164122ab8d513707c
+  metadata.gz: 221e5235470801ff068ea1fd1cf29494ac4e5c7d
+  data.tar.gz: d916df00dc140f8d88692dc3df7dd1d427522e8a
 SHA512:
-  metadata.gz: 618be24122be3c671931165e2f8c4338b3bfc9bb37553981f04f9a74986d4a67603b7ce9bdd9baca05a85947d163762e0bf317832d6fada116752ed64492c3dc
-  data.tar.gz: 64201d646e37d98b70392c8ce2db90f32c605d19b6a967a066b2f8dbc30ff1b416b1d0f966e90df5950c99a7039c4d3e96b2c697262e27cbfe5a4cf5314116b9
+  metadata.gz: 99168b0111a49424e6166a91a0586819b65df8007edfe766740df902bb922400c442ef58a52ae4611f3737d5f609b52bdabf92a24b319849d5fd3e1af669eae1
+  data.tar.gz: 12f4a20fc48a4606d9e841f5fbba75189294090f668d7f32822d236d39ebe979b871f3e2a4efc19ed395593d9ef174535989493ae25ecab29674e4ac11f8ffd2

data/bin/pa-lbaas-agent

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 'port-authority/agents/lbaas'
+PortAuthority::Agents::LBaaS.new

data/bin/pa-service-list

@@ -1,3 +1,3 @@
 #!/usr/bin/env ruby
-require 'port-authority/services'
-PortAuthority::Services::App.new.run
+require 'port-authority/tools/service_list'
+PortAuthority::Tools::ServiceList.new

data/lib/port-authority.rb

@@ -1,12 +1,4 @@
 module PortAuthority
-  module Manager
-  end
-
-  module Services
-  end
-
-  module Util
-  end
 
   module Errors
 

data/lib/port-authority/agent.rb

@@ -0,0 +1,164 @@
+require 'timeout'
+require 'digest/sha2'
+require 'json'
+require 'yaml'
+require 'port-authority'
+require 'port-authority/config'
+require 'port-authority/logger'
+require 'port-authority/etcd'
+
+module PortAuthority
+  # Scaffolding class for agents
+  class Agent
+    # Common agent process init. Contains configuration load,
+    # common signal responses and runtime variables init.
+    # Implements execution of actual agents via \+run+ method.
+    # Also handles any uncaught exceptions.
+    def initialize
+      Thread.current[:name] = 'main'                        # name main thread
+      @@_exit = false                                       # prepare exit flag
+      @@_semaphores = { log: Mutex.new }                    # init semaphores
+      @@_threads = {}                                       # init threads
+      Signal.trap('INT') { exit!(1) }                       # end immediatelly
+      Signal.trap('TERM') { end! }                          # end gracefully
+      Config.load! || exit!(1)                              # load config or die
+      begin                                                 # all-wrapping exception ;)
+        run                                                 # hook to child class
+      rescue StandardError => e
+        Logger.alert "UNCAUGHT EXCEPTION IN THREAD main! Dying!  X.X"
+        Logger.alert [' ', "#{e.class}:", e.message].join(' ')
+        e.backtrace.each {|line| Logger.debug "  #{line}"}
+        exit! 1
+      end
+    end
+
+    # Setup the agent process.
+    # Initializes logging, system process parameters,
+    # daemonizing.
+    #
+    # There are 4 optional parameters:
+    # +:name+:: \+String+ Agent name. Defaults to \+self.class.downcase+ of the child agent
+    # +:root+:: \+Bool+ Require to be ran as root. Defaults to \+false+.
+    # +:daemonize+:: \+Bool+ Daemonize the process. Defaults to \+false+.
+    # +:nice+:: \+Int+ nice of the process. Defaults to \+0+
+    def setup(args = {})
+      name = args[:name] || self.class.to_s.downcase.split('::').last
+      args[:root] ||= false
+      args[:daemonize] ||= false
+      args[:nice] ||= 0
+      Logger.init! @@_semaphores[:log]
+      Logger.info 'Starting main thread'
+      Logger.debug 'Setting process name'
+      if RUBY_VERSION >= '2.1'
+        Process.setproctitle("pa-#{name}-agent")
+      else
+        $0 = "pa-#{name}-agent"
+      end
+      if args[:root] && Process.uid != 0
+        Logger.alert 'Must run under root user!'
+        exit! 1
+      end
+      Logger.debug 'Setting CPU nice level'
+      Process.setpriority(Process::PRIO_PROCESS, 0, args[:nice])
+      if args[:daemonize]
+        Logger.info 'Daemonizing process'
+        if RUBY_VERSION < '1.9'
+          exit if fork
+          Process.setsid
+          exit if fork
+          Dir.chdir('/')
+        else
+          Process.daemon
+        end
+      end
+    end
+
+    # Has the exit flag been raised?
+    def exit?
+      @@_exit
+    end
+
+    # Raise the exit flag
+    def end!
+      @@_exit = true
+    end
+
+    # Create a named \+Mutex+ semaphore
+    def sem_create(name)
+      @@_semaphores.merge!(Hash[name.to_sym], Mutex.new)
+    end
+
+    # Create a named \+Thread+ with its \+Mutex+ semaphore.
+    # The definition includes \+&block+ of code that should run
+    # within the thread.
+    #
+    # The method requires 3 parameters:
+    # +name+:: \+Symbol+ Thread/Mutex name.
+    # +interval+:: \+Integer+ Thread loop interval.
+    # +&block+:: \+Proc+ Block of code to run.
+    def thr_create(name, interval, &block)
+      @@_semaphores.merge!(Hash[name.to_sym, Mutex.new])
+      @@_threads.merge!(Hash[name.to_sym, Thread.new do
+          Thread.current[:name] = name.to_s
+          Logger.info "Starting thread #{Thread.current[:name]}"
+          begin
+            until exit?
+              yield block
+              sleep interval
+            end
+            Logger.info "Ending thread #{Thread.current[:name]}"
+          rescue StandardError => e
+            Logger.alert "UNCAUGHT EXCEPTION IN THREAD #{Thread.current[:name]}"
+            Logger.alert [' ', "#{e.class}:", e.message].join(' ')
+            e.backtrace.each {|line| Logger.debug "  #{line}"}
+            end!
+          end
+        end
+      ])
+    end
+
+    # Run thread-safe code.
+    # The \+name+ parameter can be omitted when used
+    # from within a block of thread code. In this case
+    # the Mutex with the same \+:name+ will be used.
+    #
+    # The method accepts following parameters:
+    # +name+:: \+Symbol+ Mutex name.
+    # +&block+:: \+Proc+ Block of code to run.
+    def thr_safe(name=Thread.current[:name].to_sym, &block)
+      @@_semaphores[name.to_sym].synchronize do
+        yield block
+      end
+    end
+
+    # Start named thread.
+    # If the name is omitted, applies to all spawned threads ;)
+    def thr_start(name=nil)
+      return @@_threads[name].run if name
+      @@_threads.each_value(&:run)
+    end
+
+
+    # Wait for named thread to finish.
+    # If the name is omitted, applies to all spawned threads ;)
+    def thr_wait(name=nil)
+      return @@_threads[name].join if name
+      @@_threads.each_value(&:join)
+    end
+
+    # Run command in Bash.
+    def shellcmd(*args)
+      cmd = args.join(' ').to_s
+      cksum = Digest::SHA256.hexdigest(args.join.to_s)[0..15]
+      Logger.debug "Executing shellcommand #{cksum} - #{cmd}"
+      ret = system cmd
+      Logger.debug "Shellcommand #{cksum} returned #{ret.to_s}"
+    end
+
+    # Return hostname.
+    def hostname
+      @hostname ||= Socket.gethostname
+    end
+
+  end
+end

data/lib/port-authority/agents/lbaas.rb

@@ -0,0 +1,134 @@
+# rubocop:disable MethodLength, CyclomaticComplexity, Metrics/BlockNesting, Metrics/LineLength, Metrics/AbcSize, Metrics/PerceivedComplexity
+require 'socket'
+require 'port-authority/agent'
+require 'port-authority/mechanism/load_balancer'
+require 'port-authority/mechanism/floating_ip'
+
+module PortAuthority
+  module Agents
+    class LBaaS < PortAuthority::Agent
+      include PortAuthority::Mechanism
+
+      def run
+        setup(daemonize: Config.daemonize, nice: -10, root: true)
+        Signal.trap('HUP') { Config.load! && LoadBalancer.init! && FloatingIP.init! }
+        Signal.trap('USR1') { Logger.debug! }
+        Signal.trap('USR2') { @lb_update_hook = true }
+        @status_swarm = false
+        @etcd = PortAuthority::Etcd.cluster_connect Config.etcd
+
+        thr_create(:swarm, Config.lbaas[:swarm_interval] || Config.lbaas[:interval]) do
+          begin
+            Logger.debug 'Checking Swarm state'
+            status = @etcd.am_i_swarm_leader?
+            thr_safe { @status_swarm = status }
+            Logger.debug "I am Swarm #{status ? 'leader' : 'follower' }"
+          rescue StandardError => e
+            Logger.error [ e.class, e.message ].join(': ')
+            e.backtrace.each {|line| Logger.debug "  #{line}"}
+            thr_safe { @status_swarm = false }
+            sleep(Config.lbaas[:swarm_interval] || Config.lbaas[:interval])
+            retry unless exit?
+          end
+        end
+
+        thr_start
+
+        FloatingIP.init!
+        LoadBalancer.init!
+        LoadBalancer.container || ( LoadBalancer.pull! && LoadBalancer.create! )
+
+        Logger.debug 'Waiting for threads to gather something...'
+        sleep Config.lbaas[:interval]
+        first_cycle = true
+        status_time = Time.now.to_i - 60
+
+        until exit?
+          status_swarm = false if first_cycle
+          if @lb_update_hook
+            Logger.notice 'LoadBalancer update triggerred'
+            LoadBalancer.update!
+            @lb_update_hook = false
+            Logger.notice 'LoadBalancer update finished'
+          end
+          sleep Config.lbaas[:interval]
+          thr_safe(:swarm) { status_swarm = @status_swarm }
+          # main logic
+          if status_swarm
+            # handle FloatingIP on leader
+            Logger.debug 'I am the LEADER'
+            if FloatingIP.up?
+              Logger.debug 'Got FloatingIP, that is OK'
+            else
+              Logger.notice 'No FloatingIP here, checking whether it is free'
+              FloatingIP.arp_del!
+              if FloatingIP.reachable?
+                Logger.notice 'FloatingIP is still up! (ICMP)'
+              else
+                Logger.info 'FloatingIP is unreachable by ICMP, checking for duplicates on L2'
+                FloatingIP.arp_del!
+                if FloatingIP.duplicate?
+                  Logger.error 'FloatingIP is still assigned! (ARP)'
+                else
+                  Logger.notice 'FloatingIP is free :) assigning'
+                  FloatingIP.handle! status_swarm
+                  Logger.notice 'Notifying the network about change'
+                  FloatingIP.update_arp!
+                end
+              end
+            end
+            # handle LoadBalancer on leader
+            if LoadBalancer.up?
+              Logger.debug 'LoadBalancer is up, that is OK'
+            else
+              Logger.notice 'LoadBalancer is down, starting'
+              LoadBalancer.start!
+            end
+          else
+            # handle FloatingIP on follower
+            Logger.debug 'I am a follower'
+            if FloatingIP.up?
+              Logger.notice 'I got FloatingIP and should not, removing'
+              FloatingIP.handle! status_swarm
+              FloatingIP.arp_del!
+              Logger.notice 'Notifying the network about change'
+              FloatingIP.update_arp!
+            else
+              Logger.debug 'No FloatingIP here, that is OK'
+            end
+            # handle LoadBalancer on follower
+            if LoadBalancer.up?
+              Logger.notice 'LoadBalancer is up, stopping'
+              LoadBalancer.stop!
+            else
+              Logger.debug 'LoadBalancer is down, that is OK'
+            end
+          end # logic end
+        end
+
+        thr_wait
+
+        # remove FloatingIP on shutdown
+        if FloatingIP.up?
+          Logger.notice 'Removing FloatingIP'
+          FloatingIP.handle! false
+          FloatingIP.update_arp!
+        end
+
+        # stop LB on shutdown
+        if LoadBalancer.up?
+          Logger.notice 'Stopping LoadBalancer'
+          LoadBalancer.stop!
+        end
+
+        Logger.notice 'Exiting...'
+        exit 0
+      end
+
+      def my_ip
+        @my_ip ||= Socket.ip_address_list.detect(&:ipv4_private?).ip_address
+      end
+
+    end
+  end
+end

data/lib/port-authority/config.rb

@@ -0,0 +1,49 @@
+require 'etcd-tools/mixins/hash'
+
+module PortAuthority
+  module Config
+
+    extend self
+
+    attr_reader :_cfg
+
+    def method_missing(name, *_args, &_block)
+      return @_cfg[name.to_sym] if @_cfg[name.to_sym] != nil
+      fail(NoMethodError, "unknown configuration section #{name}", caller)
+    end
+
+    def load!
+      @_cfg = {
+        debug: false,
+        syslog: false,
+        daemonize: false,
+        etcd: {
+          endpoints: ['http://localhost:2379'],
+          timeout: 5
+        },
+        commands: {
+          arping: `which arping`.chomp,
+          arp: `which arp`.chomp,
+          iproute: `which ip`.chomp
+        }
+      }
+      files = ['/etc/port-authority.yaml', './etc/port-authority.yaml'].delete_if {|f| !File.exists?(f)}
+      dir_files = Dir['/etc/port-authority.d/**.yaml'] + Dir['./etc/port-authority.d/**.yaml']
+      files += dir_files
+      return false if files.empty?
+      files.each do |f|
+        @_cfg = @_cfg.deep_merge(YAML.load_file(f))
+      end
+      true
+    end
+
+    def dump
+      self._cfg
+    end
+
+    def to_yaml
+      self._cfg.to_yaml.to_s
+    end
+
+  end
+end

data/lib/port-authority/etcd.rb

@@ -0,0 +1,47 @@
+require 'etcd-tools'
+require 'etcd-tools/mixins'
+
+module PortAuthority
+  class Etcd < Etcd::Client
+
+    def self.cluster_connect(config)
+      endpoints = config[:endpoints].map {|ep| Hash[[:host, :port].zip(ep.match(/^(?:https?:\/\/)?([0-9a-zA-Z\.-_]+):([0-9]+)\/?/).captures)]}
+      timeout = config[:timeout].to_i
+      PortAuthority::Etcd.new(cluster: endpoints, read_timeout: timeout)
+    end
+
+    def self.shell_cluster_connect(env, timeout = 2)
+      env.split(',').each do |u|
+        (host, port) = u.gsub(/^https?:\/\//, '').gsub(/\/$/, '').split(':')
+        etcd = PortAuthority::Etcd.new(cluster: [{ host: host, port: port }], read_timeout: timeout)
+        return etcd if etcd.healthy?
+        next
+      end
+      raise Etcd::ClusterConnectError
+    end
+
+
+    def swarm_leader
+      get('/_pa/docker/swarm/leader').value.split(':').first
+    end
+
+    def am_i_swarm_leader?
+      Socket.ip_address_list.map(&:ip_address).member?(swarm_leader)
+    end
+
+    def swarm_overlay_id(name)
+      get_hash('/_pa/docker/network/v1.0/network').each_value do |network|
+        return network['id'] if network['name'] == name
+      end
+    end
+
+    def swarm_list_services(network, service_name='.*')
+      services = {}
+      self.get_hash("/_pa/docker/network/v1.0/endpoint/#{swarm_overlay_id(network)}").each_value do |container|
+        next unless Regexp.new(service_name).match container['name']
+        services = { container['name'] => { 'id' => container['id'], 'ip' => container['ep_iface']['addr'].sub(/\/[0-9]+$/, ''), 'ports' => container['exposed_ports'].map { |port| port['Port'] } } }
+      end
+      services
+    end
+  end
+end

data/lib/port-authority/logger.rb

@@ -0,0 +1,56 @@
+# rubocop:disable Metrics/LineLength, Metrics/AbcSize, Metrics/MethodLength
+require 'syslog'
+
+module PortAuthority
+  module Logger
+
+    extend self
+
+    def init!(s)
+      @_s = s
+      @debug = Config.debug
+      Syslog.open($0, Syslog::LOG_PID, Syslog::LOG_DAEMON) if Config.syslog
+    end
+
+    def debug!
+      @debug = !@debug
+    end
+
+    def debug(message)
+      log :debug, message if @debug
+    end
+
+    def method_missing(name, *args, &_block)
+      if name
+        return log(name.to_sym, args[0])
+      else
+        fail(NoMethodError, "Unknown Logger method '#{name}'", caller)
+      end
+    end
+
+    def log(lvl, msg)
+      if Config.syslog
+        case lvl
+        when :debug
+          l = Syslog::LOG_DEBUG
+        when :info
+          l = Syslog::LOG_INFO
+        when :notice
+          l = Syslog::LOG_NOTICE
+        when :error
+          l = Syslog::LOG_ERR
+        when :alert
+          l = Syslog::LOG_ALERT
+        end
+        @_s.synchronize do
+          Syslog.log(l, "(%s) %s", Thread.current[:name], msg.to_s)
+        end
+      else
+        @_s.synchronize do
+          $stdout.puts [Time.now.to_s, sprintf('%-6.6s', lvl.to_s.upcase), "(#{Thread.current[:name]})", msg.to_s].join(' ')
+          $stdout.flush
+        end
+      end
+    end
+  end
+end