fault_tolerant_router 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c619776dd495528564bc6177a684c7ab5f1da84
-  data.tar.gz: 6b0434ab1d89c616f0e63e22600ffcb6c6f818cd
+  metadata.gz: 5d0eeae84605fd65228ba43ec1429ce65f579ed2
+  data.tar.gz: 0bfee9df0185f24b895d7f20330eb4fdd4381345
 SHA512:
-  metadata.gz: 102eb31be1f36deb2a01c195280aebb8d13b371c24c9e538845604be7dc310b54772926abe3c0f2e029983c1bdbd09a383157dbfaf7fcf59c9374d321d73b499
-  data.tar.gz: 713419088384c7486c4994cfb21afa6272a110258cf067cb5ec9d7c441017cf3d2ccf0f6327d04e2f1eef5d738f0dbeb04137a945db60e070644e82fc8e025b3
+  metadata.gz: 0f189d0a607af22820ad949951dfd012d9f2223b4de417c16b1f11c109a45849516da27d0372dc5f6415e3219edc5d27c10940168fd170c01030e31016eb3c83
+  data.tar.gz: 9463c6b3a497017102b63566553d9b47c4fccdd0e454dc839e8096bacaea9506ca07004e84c869a79d5f11396af6fc15f1d30827a9738b984c06412ac0c6f6c2

data/README.md

@@ -86,8 +86,9 @@ If you want a quick and dirty way to run the program in background, just add an
 The fault_tolerant_router.conf configuration file is in [YAML](http://en.wikipedia.org/wiki/YAML) format. Here is the explanation of the options:
 * **uplinks**: Array of uplinks. The example configuration has 3 uplinks, but you can have from 2 to as many as you wish.
   * **interface**: The network interface where the uplink is connected. Until today Fault Tolerant Router has always been used with each uplink on it's own physical interface, never tried it with VLAN interfaces (it's in the to do list).
-  * **ip**: Primary IP address of the network interface. You can have more than one IP address assigned to the interface, just specify here the primary one that will be used as standard SNAT source.
-  * **gateway**: The uplink gateway, usually the provider's router IP address.
+  * **type**: Specify *static* for any kind of static IP interface or *ppp* for a PPP dynamic IP interface.
+  * **ip**: Primary IP address of the network interface. You can have more than one IP address assigned to the interface, just specify here the primary one that will be used as standard SNAT source. Omit this parameter in case of a PPP dynamic IP interface.
+  * **gateway**: The uplink gateway, usually the provider's router IP address. Omit this parameter in case of a PPP dynamic IP interface.
   * **description**: Uplink name, used in notifications.
   * **weight**: Optional parameter, it's the preference to assign to this uplink when choosing one for a new outgoing connection. Use when you have uplinks with different bandwidths. See http://www.policyrouting.org/PolicyRoutingBook/ONLINE/CH05.web.html
   * **default_route**: Optional parameter, default value is *true*. If set to *false* the uplink is excluded from the *multipath routing*, i.e. the uplink will never be selected when choosing one for a new outgoing connection. There's an exception to this if some kind of outgoing connection is forced to pass through this uplink, see [Iptables rules](#iptables-rules) section. Note this parameter only affects outgoing connections, even if set to *false* incoming connections are still possible. Use cases to set it to *false*:
@@ -169,7 +170,7 @@ Documentation is included as comments in the output, here is a dump using the st
 #Example Provider 2
 [0:0] -A PREROUTING -i eth2 -m conntrack --ctstate NEW -j CONNMARK --set-mark 2
 #Example Provider 3
-[0:0] -A PREROUTING -i eth3 -m conntrack --ctstate NEW -j CONNMARK --set-mark 3
+[0:0] -A PREROUTING -i ppp0 -m conntrack --ctstate NEW -j CONNMARK --set-mark 3
 
 #New outbound connections: mark the connection with the outgoing interface
 #(chosen by the multipath routing).
@@ -179,7 +180,7 @@ Documentation is included as comments in the output, here is a dump using the st
 #Example Provider 2
 [0:0] -A POSTROUTING -o eth2 -m conntrack --ctstate NEW -j CONNMARK --set-mark 2
 #Example Provider 3
-[0:0] -A POSTROUTING -o eth3 -m conntrack --ctstate NEW -j CONNMARK --set-mark 3
+[0:0] -A POSTROUTING -o ppp0 -m conntrack --ctstate NEW -j CONNMARK --set-mark 3
 
 COMMIT
 
@@ -203,7 +204,7 @@ COMMIT
 #Example Provider 2
 #[0:0] -A PREROUTING -i eth2 -d 2.0.0.2 -j DNAT --to-destination XXX.XXX.XXX.XXX
 #Example Provider 3
-#[0:0] -A PREROUTING -i eth3 -d 3.0.0.2 -j DNAT --to-destination XXX.XXX.XXX.XXX
+#[0:0] -A PREROUTING -i ppp0 -j DNAT --to-destination XXX.XXX.XXX.XXX
 
 #SNAT: LAN/DMZ --> WAN. Force an outgoing connection to use a specific source
 #address instead of the default one of the outgoing interface. Of course this
@@ -220,7 +221,7 @@ COMMIT
 #Example Provider 2
 #[0:0] -A POSTROUTING -s XXX.XXX.XXX.XXX -o eth2 -j SNAT --to-source YYY.YYY.YYY.YYY
 #Example Provider 3
-#[0:0] -A POSTROUTING -s XXX.XXX.XXX.XXX -o eth3 -j SNAT --to-source YYY.YYY.YYY.YYY
+#[0:0] -A POSTROUTING -s XXX.XXX.XXX.XXX -o ppp0 -j SNAT --to-source YYY.YYY.YYY.YYY
 
 #SNAT: LAN --> WAN
 
@@ -229,7 +230,7 @@ COMMIT
 #Example Provider 2
 [0:0] -A POSTROUTING -o eth2 -j SNAT --to-source 2.0.0.2
 #Example Provider 3
-[0:0] -A POSTROUTING -o eth3 -j SNAT --to-source 3.0.0.2
+[0:0] -A POSTROUTING -o ppp0 -j MASQUERADE
 
 COMMIT
 
@@ -251,10 +252,10 @@ COMMIT
 
 [0:0] -A FORWARD -i eth0 -o eth1 -j LAN_WAN
 [0:0] -A FORWARD -i eth0 -o eth2 -j LAN_WAN
-[0:0] -A FORWARD -i eth0 -o eth3 -j LAN_WAN
+[0:0] -A FORWARD -i eth0 -o ppp0 -j LAN_WAN
 [0:0] -A FORWARD -i eth1 -o eth0 -j WAN_LAN
 [0:0] -A FORWARD -i eth2 -o eth0 -j WAN_LAN
-[0:0] -A FORWARD -i eth3 -o eth0 -j WAN_LAN
+[0:0] -A FORWARD -i ppp0 -o eth0 -j WAN_LAN
 
 #This is just a very basic example, add your own rules for the FORWARD chain.
 

data/bin/fault_tolerant_router

@@ -4,11 +4,14 @@ require 'optparse'
 require 'net/smtp'
 require 'mail'
 require 'logger'
+require 'socket'
 require 'yaml'
 require 'fault_tolerant_router/version'
 require 'fault_tolerant_router/generate_config'
 require 'fault_tolerant_router/generate_iptables'
 require 'fault_tolerant_router/monitor'
+require 'fault_tolerant_router/uplink'
+require 'fault_tolerant_router/uplinks'
 
 options = {
     configuration_file: '/etc/fault_tolerant_router.conf',
@@ -22,14 +25,14 @@ parser = OptionParser.new do |opts|
   opts.separator ''
   opts.separator '         generate_config: save an example configuration'
   opts.separator '         generate_iptables: generate an "iptables-save" configuration'
-  opts.separator '         email_test: send a test email to verify the correctness of smtp parameters'
+  opts.separator '         email_test: send a test email to verify the correctness of SMTP parameters'
   opts.separator '         monitor: monitor uplinks and change routing accordingly'
   opts.separator ''
+  opts.separator 'OPTIONS:'
   opts.on('--config=FILE', 'Configuration file (default /etc/fault_tolerant_router.conf)') do |configuration_file|
     options[:configuration_file] = configuration_file
   end
-  opts.separator ''
-  opts.separator '"monitor" specific options:'
+  opts.separator '        "monitor" specific options:'
   opts.on('--debug', 'Print debug output') do |debug|
     options[:debug] = debug
   end
@@ -49,7 +52,7 @@ rescue OptionParser::ParseError
 end
 
 DEMO = options[:demo]
-#activate debug if we are in demo mode
+#activate debug if in demo mode
 DEBUG = options[:debug] || DEMO
 
 unless ARGV.size == 1 && %w(generate_config generate_iptables email_test monitor).include?(ARGV[0])
@@ -68,14 +71,13 @@ unless File.exists?(options[:configuration_file])
 end
 
 config = YAML.load_file(options[:configuration_file])
-UPLINKS = config['uplinks'].map { |uplink| Hash[uplink.map { |k, v| [k.to_sym, v] }] }
 LAN_INTERFACE = config['downlinks']['lan']
 DMZ_INTERFACE = config['downlinks']['dmz']
 TEST_IPS = config['tests']['ips']
 REQUIRED_SUCCESSFUL_TESTS = config['tests']['required_successful']
 PING_RETRIES = config['tests']['ping_retries']
 TEST_INTERVAL = config['tests']['interval']
-LOG_FILE = config['log']['log_file']
+LOG_FILE = config['log']['file']
 LOG_MAX_SIZE = config['log']['max_size']
 LOG_OLD_FILES = config['log']['old_files']
 SEND_EMAIL = config['email']['send']
@@ -85,6 +87,7 @@ SMTP_PARAMETERS = Hash[config['email']['smtp_parameters'].map { |k, v| [k.to_sym
 BASE_TABLE = config['base_table']
 BASE_PRIORITY = config['base_priority']
 BASE_FWMARK = config['base_fwmark']
+UPLINKS = Uplinks.new(config['uplinks'])
 
 case ARGV[0]
   when 'generate_iptables'
@@ -94,7 +97,7 @@ case ARGV[0]
       send_email('fault_tolerant_router test')
       puts "Test email sent to #{EMAIL_RECIPIENTS.join(', ')}"
     rescue Exception => e
-      puts "Problem sending email: #{e}"
+      puts "Error sending email: #{e}"
     end
   else
     monitor

data/lib/fault_tolerant_router/generate_config.rb

@@ -12,6 +12,7 @@ def generate_config(file_path)
 #add as many uplinks as needed
 uplinks:
 - interface: eth1
+  type: static
   ip: 1.0.0.2
   gateway: 1.0.0.1
   description: Example Provider 1
@@ -20,6 +21,7 @@ uplinks:
   #optional parameter, default is true
   default_route: true
 - interface: eth2
+  type: static
   ip: 2.0.0.2
   gateway: 2.0.0.1
   description: Example Provider 2
@@ -27,9 +29,8 @@ uplinks:
   weight: 2
   #optional parameter, default is true
   default_route: true
-- interface: eth3
-  ip: 3.0.0.2
-  gateway: 3.0.0.1
+- interface: ppp0
+  type: ppp
   description: Example Provider 3
   #optional parameter
   weight: 1

data/lib/fault_tolerant_router/generate_iptables.rb

@@ -21,10 +21,10 @@ def generate_iptables
 #    --sport, etc.
 
 END
-  UPLINKS.each_with_index do |uplink, i|
-    puts "##{uplink[:description]}"
-    puts "#[0:0] -A PREROUTING -i #{LAN_INTERFACE} -m conntrack --ctstate NEW -p tcp --dport XXX -j CONNMARK --set-mark #{BASE_FWMARK + i}"
-    puts "#[0:0] -A PREROUTING -i #{DMZ_INTERFACE} -m conntrack --ctstate NEW -p tcp --dport XXX -j CONNMARK --set-mark #{BASE_FWMARK + i}" if DMZ_INTERFACE
+  UPLINKS.each do |uplink|
+    puts "##{uplink.description}"
+    puts "#[0:0] -A PREROUTING -i #{LAN_INTERFACE} -m conntrack --ctstate NEW -p tcp --dport XXX -j CONNMARK --set-mark #{uplink.fwmark}"
+    puts "#[0:0] -A PREROUTING -i #{DMZ_INTERFACE} -m conntrack --ctstate NEW -p tcp --dport XXX -j CONNMARK --set-mark #{uplink.fwmark}" if DMZ_INTERFACE
   end
   puts <<END
 
@@ -47,9 +47,9 @@ END
 #New inbound connections: mark the connection with the incoming interface.
 
 END
-  UPLINKS.each_with_index do |uplink, i|
-    puts "##{uplink[:description]}"
-    puts "[0:0] -A PREROUTING -i #{uplink[:interface]} -m conntrack --ctstate NEW -j CONNMARK --set-mark #{BASE_FWMARK + i}"
+  UPLINKS.each do |uplink|
+    puts "##{uplink.description}"
+    puts "[0:0] -A PREROUTING -i #{uplink.interface} -m conntrack --ctstate NEW -j CONNMARK --set-mark #{uplink.fwmark}"
   end
   puts <<END
 
@@ -57,9 +57,9 @@ END
 #(chosen by the multipath routing).
 
 END
-  UPLINKS.each_with_index do |uplink, i|
-    puts "##{uplink[:description]}"
-    puts "[0:0] -A POSTROUTING -o #{uplink[:interface]} -m conntrack --ctstate NEW -j CONNMARK --set-mark #{BASE_FWMARK + i}"
+  UPLINKS.each do |uplink|
+    puts "##{uplink.description}"
+    puts "[0:0] -A POSTROUTING -o #{uplink.interface} -m conntrack --ctstate NEW -j CONNMARK --set-mark #{uplink.fwmark}"
   end
   puts <<END
 
@@ -82,8 +82,12 @@ COMMIT
 
 END
   UPLINKS.each do |uplink|
-    puts "##{uplink[:description]}"
-    puts "#[0:0] -A PREROUTING -i #{uplink[:interface]} -d #{uplink[:ip]} -j DNAT --to-destination XXX.XXX.XXX.XXX"
+    puts "##{uplink.description}"
+    if uplink.type == :ppp
+      puts "#[0:0] -A PREROUTING -i #{uplink.interface} -j DNAT --to-destination XXX.XXX.XXX.XXX"
+    else
+      puts "#[0:0] -A PREROUTING -i #{uplink.interface} -d #{uplink.ip} -j DNAT --to-destination XXX.XXX.XXX.XXX"
+    end
   end
   puts <<END
 
@@ -99,8 +103,8 @@ END
 
 END
   UPLINKS.each do |uplink|
-    puts "##{uplink[:description]}"
-    puts "#[0:0] -A POSTROUTING -s XXX.XXX.XXX.XXX -o #{uplink[:interface]} -j SNAT --to-source YYY.YYY.YYY.YYY"
+    puts "##{uplink.description}"
+    puts "#[0:0] -A POSTROUTING -s XXX.XXX.XXX.XXX -o #{uplink.interface} -j SNAT --to-source YYY.YYY.YYY.YYY"
   end
   puts <<END
 
@@ -108,8 +112,12 @@ END
 
 END
   UPLINKS.each do |uplink|
-    puts "##{uplink[:description]}"
-    puts "[0:0] -A POSTROUTING -o #{uplink[:interface]} -j SNAT --to-source #{uplink[:ip]}"
+    puts "##{uplink.description}"
+    if uplink.type == :ppp
+      puts "[0:0] -A POSTROUTING -o #{uplink.interface} -j MASQUERADE"
+    else
+      puts "[0:0] -A POSTROUTING -o #{uplink.interface} -j SNAT --to-source #{uplink.ip}"
+    end
   end
   puts <<END
 
@@ -141,17 +149,17 @@ END
 
 END
   UPLINKS.each do |uplink|
-    puts "[0:0] -A FORWARD -i #{LAN_INTERFACE} -o #{uplink[:interface]} -j LAN_WAN"
+    puts "[0:0] -A FORWARD -i #{LAN_INTERFACE} -o #{uplink.interface} -j LAN_WAN"
   end
   UPLINKS.each do |uplink|
-    puts "[0:0] -A FORWARD -i #{uplink[:interface]} -o #{LAN_INTERFACE} -j WAN_LAN"
+    puts "[0:0] -A FORWARD -i #{uplink.interface} -o #{LAN_INTERFACE} -j WAN_LAN"
   end
   if DMZ_INTERFACE
     UPLINKS.each do |uplink|
-      puts "[0:0] -A FORWARD -i #{DMZ_INTERFACE} -o #{uplink[:interface]} -j DMZ_WAN"
+      puts "[0:0] -A FORWARD -i #{DMZ_INTERFACE} -o #{uplink.interface} -j DMZ_WAN"
     end
     UPLINKS.each do |uplink|
-      puts "[0:0] -A FORWARD -i #{uplink[:interface]} -o #{DMZ_INTERFACE} -j WAN_DMZ"
+      puts "[0:0] -A FORWARD -i #{uplink.interface} -o #{DMZ_INTERFACE} -j WAN_DMZ"
     end
   end
   puts <<END

data/lib/fault_tolerant_router/monitor.rb

@@ -1,36 +1,13 @@
-def command(c)
-  `#{c}` unless DEMO
-  puts "Command: #{c}" if DEBUG
-end
-
-def ping(ip, source)
-  if DEMO
-    sleep 0.1
-    rand(3) > 0
-  else
-    `ping -n -c 1 -W 2 -I #{source} #{ip}`
-    $?.to_i == 0
+def command(input)
+  input = [input] if input.is_a?(String)
+  input.each do |c|
+    `#{c}` unless DEMO
+    puts "Command: #{c}" if DEBUG
   end
 end
 
-def set_default_route
-  #find the enabled uplinks
-  enabled_uplinks = UPLINKS.find_all { |uplink| uplink[:enabled] }
-  #do not use balancing if there is just one enabled uplink
-  if enabled_uplinks.size == 1
-    nexthops = "via #{enabled_uplinks.first[:gateway]}"
-  else
-    nexthops = enabled_uplinks.collect do |uplink|
-      #the "weight" parameter is optional
-      weight = uplink[:weight] ? " weight #{uplink[:weight]}" : ''
-      "nexthop via #{uplink[:gateway]}#{weight}"
-    end
-    nexthops = nexthops.join(' ')
-  end
-  #set the route for first packet of outbound connections
-  command "ip route replace table #{BASE_TABLE + UPLINKS.size} default #{nexthops}"
-  #apply the routing changes
-  command 'ip route flush cache'
+def log(logger, messages)
+  logger.warn(messages.join('; ')) if messages.any?
 end
 
 def send_email(body)
@@ -45,141 +22,34 @@ end
 
 def monitor
   logger = Logger.new(LOG_FILE, LOG_OLD_FILES, LOG_MAX_SIZE)
-
-  #enable all the uplinks
-  UPLINKS.each do |uplink|
-    uplink[:working] = true
-    uplink[:default_route] ||= uplink[:default_route].nil?
-    uplink[:enabled] = uplink[:default_route]
-  end
-
-  #clean all previous configurations, try to clean more than needed (double) to avoid problems in case of changes in the
-  #number of uplinks between different executions
-  ((UPLINKS.size * 2 + 1) * 2).times do |i|
-    command "ip rule del priority #{BASE_PRIORITY + i} &> /dev/null"
-  end
-  ((UPLINKS.size + 1) * 2).times do |i|
-    command "ip route del table #{BASE_TABLE + i} &> /dev/null"
-  end
-
-  #enable IP forwarding
-  command 'echo 1 > /proc/sys/net/ipv4/ip_forward'
-
-  #disable "reverse path filtering" on the uplink interfaces
-  command 'echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter'
-  UPLINKS.each do |uplink|
-    command "echo 2 > /proc/sys/net/ipv4/conf/#{uplink[:interface]}/rp_filter"
-  end
-
-  #- locally generated packets having as source ip the ethX ip
-  #- returning packets of inbound connections coming from ethX
-  #- non-first packets of outbound connections for which the first packet has been sent to ethX via multipath routing
-  UPLINKS.each_with_index do |uplink, i|
-    command "ip route add table #{BASE_TABLE + i} default via #{uplink[:gateway]} src #{uplink[:ip]}"
-    command "ip rule add priority #{BASE_PRIORITY + i} from #{uplink[:ip]} lookup #{BASE_TABLE + i}"
-    command "ip rule add priority #{BASE_PRIORITY + UPLINKS.size + i} fwmark #{BASE_FWMARK + i} lookup #{BASE_TABLE + i}"
-  end
-  #first packet of outbound connections
-  command "ip rule add priority #{BASE_PRIORITY + UPLINKS.size * 2} from all lookup #{BASE_TABLE + UPLINKS.size}"
-  set_default_route
+  command UPLINKS.initialize_routing_commands
 
   loop do
-    #for each uplink...
-    UPLINKS.each do |uplink|
-      #set current "working" state as the previous one
-      uplink[:previously_working] = uplink[:working]
-      #set current "enabled" state as the previous one
-      uplink[:previously_enabled] = uplink[:enabled]
-      uplink[:successful_tests] = 0
-      uplink[:unsuccessful_tests] = 0
-      #for each test (in random order)...
-      TEST_IPS.shuffle.each_with_index do |test, i|
-        successful_test = false
-        #retry for several times...
-        PING_RETRIES.times do
-          if DEBUG
-            print "Uplink #{uplink[:description]}: ping #{test}... "
-            STDOUT.flush
-          end
-          if ping(test, uplink[:ip])
-            successful_test = true
-            puts 'ok' if DEBUG
-            #avoid more pings to the same ip after a successful one
-            break
-          else
-            puts 'error' if DEBUG
-          end
-        end
-        if successful_test
-          uplink[:successful_tests] += 1
-        else
-          uplink[:unsuccessful_tests] += 1
-        end
-        #if not currently doing the last test...
-        if i + 1 < TEST_IPS.size
-          if uplink[:successful_tests] >= REQUIRED_SUCCESSFUL_TESTS
-            puts "Uplink #{uplink[:description]}: avoiding more tests because there are enough positive ones" if DEBUG
-            break
-          elsif TEST_IPS.size - uplink[:unsuccessful_tests] < REQUIRED_SUCCESSFUL_TESTS
-            puts "Uplink #{uplink[:description]}: avoiding more tests because too many have been failed" if DEBUG
-            break
-          end
-        end
-      end
-      uplink[:working] = uplink[:successful_tests] >= REQUIRED_SUCCESSFUL_TESTS
-      uplink[:enabled] = uplink[:working] && uplink[:default_route]
-    end
-
-    #only consider uplinks flagged as default route
-    if UPLINKS.find_all { |uplink| uplink[:default_route] }.all? { |uplink| !uplink[:working] }
-      UPLINKS.find_all { |uplink| uplink[:default_route] }.each { |uplink| uplink[:enabled] = true }
-      puts 'No uplink seems to be working, enabling all of them' if DEBUG
-    end
-
-    UPLINKS.each do |uplink|
-      description = case
-                      when uplink[:enabled] && !uplink[:previously_enabled] then
-                        ', enabled'
-                      when !uplink[:enabled] && uplink[:previously_enabled] then
-                        ', disabled'
-                      else
-                        ''
-                    end
-      puts "Uplink #{uplink[:description]}: #{uplink[:successful_tests]} successful tests, #{uplink[:unsuccessful_tests]} unsuccessful tests#{description}"
-    end if DEBUG
-
-    #set a new default route if there are changes between the previous and the current uplinks situation
-    set_default_route if UPLINKS.any? { |uplink| uplink[:enabled] != uplink[:previously_enabled] }
-
-    if UPLINKS.any? { |uplink| uplink[:working] != uplink[:previously_working] }
-      body = ''
-      UPLINKS.each do |uplink|
-        body += "Uplink #{uplink[:description]}: #{uplink[:previously_working] ? 'up' : 'down'}"
-        if uplink[:previously_working] == uplink[:working]
-          body += "\n"
-        else
-          body += " --> #{uplink[:working] ? 'up' : 'down'}\n"
-        end
-      end
-
-      logger.warn(body.gsub("\n", ';'))
-
-      if SEND_EMAIL
-        begin
-          send_email(body)
-        rescue Exception => e
-          puts "Problem sending email: #{e}" if DEBUG
-          logger.error("Problem sending email: #{e}")
-        end
+    routing_commands, ip_change_messages = UPLINKS.detect_ip_changes!
+    command routing_commands
+    log(logger, ip_change_messages)
+
+    routing_commands, up_state_change_messages, all_default_route_uplinks_down = UPLINKS.test_routing!
+    command routing_commands
+    log(logger, up_state_change_messages)
+
+    if SEND_EMAIL && (ip_change_messages.any? || up_state_change_messages.any?)
+      begin
+        send_email((ip_change_messages + up_state_change_messages).join("\n"))
+      rescue Exception => e
+        puts "Problem sending email: #{e}" if DEBUG
+        logger.error("Problem sending email: #{e}")
       end
     end
 
-    if DEMO
-      puts "Waiting just 5 seconds because we are in demo mode, otherwise would wait #{TEST_INTERVAL} seconds..."
+    if all_default_route_uplinks_down
+      puts 'No waiting, because all of the default route uplinks are down' if DEBUG
+    elsif DEMO
+      puts "Waiting just 5 seconds because in demo mode, otherwise would be #{TEST_INTERVAL} seconds..." if DEBUG
       sleep 5
     else
       puts "Waiting #{TEST_INTERVAL} seconds..." if DEBUG
       sleep TEST_INTERVAL
     end
   end
-end
+end

data/lib/fault_tolerant_router/uplink.rb

@@ -0,0 +1,164 @@
+class Uplink
+  attr_reader :default_route, :description, :fwmark, :gateway, :id, :interface, :ip, :priority1, :table, :type, :up, :weight
+  attr_accessor :priority2, :routing
+
+  def initialize(config, id)
+    @id = id
+    @priority1 = BASE_PRIORITY + @id
+    @table = BASE_TABLE + @id
+    @fwmark = BASE_FWMARK + @id
+    @interface = config['interface']
+    raise "Uplink interface not specified: #{config}" unless @interface
+    @type = case config['type']
+              when 'static'
+                :static
+              when 'ppp'
+                :ppp
+              else
+                raise "Uplink type not valid: #{config}"
+            end
+    @description = config['description']
+    raise "Uplink description not specified: #{config}" unless @description
+    @weight = config['weight']
+    @default_route = config['default_route'].nil? ? true : config['default_route']
+
+    #a new uplink is supposed to be up
+    @up = true
+    #a new uplink starts as routing if it's marked as a default route
+    @routing = @default_route
+
+    if @type == :static
+      @ip = config['ip']
+      raise "Uplink IP address not specified: #{config}" unless @ip
+      @gateway = config['gateway']
+      raise "Uplink gateway not specified: #{config}" unless @gateway
+    else
+      detect_ppp_ips!
+      puts "Uplink #{@description}: initialized with [ip: #{@ip}, gateway: #{@gateway}]" if DEBUG
+    end
+  end
+
+  def detect_ppp_ips!
+    @previous_ip = @ip
+    @previous_gateway = @gateway
+    if DEMO
+      @ip = %w(3.0.0.101 3.0.0.102).sample
+      @gateway = %w(3.0.0.1 3.0.0.2).sample
+    else
+      ifaddr = Socket.getifaddrs.find { |i| i.name == @interface && i.addr && i.addr.ipv4? }
+      if ifaddr
+        @ip = ifaddr.addr.ip_address
+        @gateway = ifaddr.dstaddr.ip_address
+      else
+        #todo: what to do if it happens?
+        raise 'PPP IP address not found'
+      end
+    end
+  end
+
+  def detect_ip_changes!
+    commands = []
+    need_default_route_update = false
+    message = nil
+    if @type == :ppp
+      detect_ppp_ips!
+      if (@previous_ip != @ip) || (@previous_gateway != @gateway)
+        message = "Uplink #{@description}: IP change [ip: #{@previous_ip}, gateway: #{@previous_gateway}] --> [ip: #{@ip}, gateway: #{@gateway}]"
+        puts message if DEBUG
+        commands = [
+            [
+                "ip rule del priority #{@priority1}",
+                "ip rule del priority #{@priority2}"
+            ],
+            route_add_commands
+        ].flatten
+      end
+      need_default_route_update = @routing && (@previous_gateway != @gateway)
+    end
+    [commands, need_default_route_update, message]
+  end
+
+  def ping(ip_address)
+    if DEMO
+      sleep 0.1
+      rand(3) > 0
+    else
+      `ping -n -c 1 -W 2 -I #{@ip} #{ip_address}`
+      $?.to_i == 0
+    end
+  end
+
+  def test_routing!
+    #save current state
+    @previously_up = @up
+    @previously_routing = @routing
+
+    @successful_tests = 0
+    @unsuccessful_tests = 0
+
+    #for each test (in random order)...
+    TEST_IPS.shuffle.each_with_index do |test, i|
+      successful_test = false
+
+      #retry for several times...
+      PING_RETRIES.times do
+        if DEBUG
+          print "Uplink #{@description}: ping #{test}... "
+          STDOUT.flush
+        end
+        if ping(test)
+          successful_test = true
+          puts 'ok' if DEBUG
+          #avoid more pings to the same ip after a successful one
+          break
+        else
+          puts 'error' if DEBUG
+        end
+      end
+
+      if successful_test
+        @successful_tests += 1
+      else
+        @unsuccessful_tests += 1
+      end
+
+      #if not currently doing the last test...
+      if i + 1 < TEST_IPS.size
+        if @successful_tests >= REQUIRED_SUCCESSFUL_TESTS
+          puts "Uplink #{@description}: avoiding more tests because there are enough positive ones" if DEBUG
+          break
+        elsif TEST_IPS.size - @unsuccessful_tests < REQUIRED_SUCCESSFUL_TESTS
+          puts "Uplink #{@description}: avoiding more tests because too many have been failed" if DEBUG
+          break
+        end
+      end
+
+    end
+
+    @up = @successful_tests >= REQUIRED_SUCCESSFUL_TESTS
+    up_state_changed = @up != @previously_up
+    @routing = @up && @default_route
+    routing_state_changed = @routing != @previously_routing
+
+    state = @previously_up ? 'up' : 'down'
+    state += " --> #{@up ? 'up' : 'down'}" if up_state_changed
+    routing = @previously_routing ? 'enabled' : 'disabled'
+    routing += " --> #{@routing ? 'enabled' : 'disabled'}" if routing_state_changed
+    message = "Uplink #{@description}: #{state}"
+    puts "Uplink #{@description}: #{@successful_tests} successful tests, #{@unsuccessful_tests} unsuccessful tests, state #{state}, routing #{routing}" if DEBUG
+
+    [up_state_changed, routing_state_changed, message]
+  end
+
+  def route_add_commands
+    #- locally generated packets having as source ip the ethX ip
+    #- returning packets of inbound connections coming from ethX
+    #- non-first packets of outbound connections for which the first packet has been sent to ethX via multipath routing
+    [
+        "ip route replace table #{table} default via #{@gateway} src #{@ip}",
+        "ip rule add priority #{@priority1} from #{@ip} lookup #{table}",
+        "ip rule add priority #{@priority2} fwmark #{fwmark} lookup #{table}"
+    ]
+  end
+
+end

data/lib/fault_tolerant_router/uplinks.rb

@@ -0,0 +1,118 @@
+class Uplinks
+
+  def initialize(config)
+    @uplinks = config.each_with_index.map { |uplink, i| Uplink.new(uplink, i) }
+    @uplinks.each { |uplink| uplink.priority2 = BASE_PRIORITY + @uplinks.size + uplink.id }
+  end
+
+  def each
+    @uplinks.each { |uplink| yield uplink }
+  end
+
+  def initialize_routing_commands
+    commands = []
+    priorities = @uplinks.map { |uplink| [uplink.priority1, uplink.priority2] }.flatten.minmax
+    tables = @uplinks.map { |uplink| uplink.table }.minmax
+
+    #enable IP forwarding
+    commands += ['echo 1 > /proc/sys/net/ipv4/ip_forward']
+
+    #clean all previous configurations, try to clean more than needed (double) to avoid problems in case of changes in the
+    #number of uplinks between different executions
+    ((priorities.max - priorities.min + 2) * 2).times { |i| commands += ["ip rule del priority #{priorities.min + i} &> /dev/null"] }
+    ((tables.max - tables.min + 2) * 2).times { |i| commands += ["ip route del table #{tables.min + i} &> /dev/null"] }
+
+    #disable "reverse path filtering" on the uplink interfaces
+    commands += ['echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter']
+    commands += @uplinks.map { |uplink| "echo 2 > /proc/sys/net/ipv4/conf/#{uplink.interface}/rp_filter" }
+
+    #set uplinks routes
+    commands += @uplinks.map { |uplink| uplink.route_add_commands }
+
+    #rule for first packet of outbound connections
+    commands += ["ip rule add priority #{priorities.max + 1} from all lookup #{tables.max + 1}"]
+
+    #set default route
+    commands += set_default_route_commands
+
+    #apply the routing changes
+    commands += ['ip route flush cache']
+
+    commands.flatten
+  end
+
+  def set_default_route_commands
+    routing_uplinks = @uplinks.find_all { |uplink| uplink.routing }
+
+    #do not use balancing if there is just one routing uplink
+    if routing_uplinks.size == 1
+      nexthops = "via #{routing_uplinks.first.gateway}"
+    else
+      nexthops = routing_uplinks.map do |uplink|
+        #the "weight" parameter is optional
+        tail = uplink.weight ? " weight #{uplink.weight}" : ''
+        "nexthop via #{uplink.gateway}#{tail}"
+      end
+      nexthops = nexthops.join(' ')
+    end
+    #set the route for first packet of outbound connections
+    ["ip route replace table #{@uplinks.map { |uplink| uplink.table }.max + 1} default #{nexthops}"]
+  end
+
+  def detect_ip_changes!
+    commands = []
+    need_default_route_update = false
+    messages = []
+
+    @uplinks.each do |uplink|
+      c, n, m = uplink.detect_ip_changes!
+      commands += c if c.any?
+      need_default_route_update ||= n
+      messages << m if m
+    end
+
+    if need_default_route_update
+      puts 'Will update default route because some of its gateways changed' if DEBUG
+      commands += set_default_route_commands
+    end
+
+    #apply the routing changes, in any
+    commands += ['ip route flush cache'] if commands.any?
+
+    [commands, messages]
+  end
+
+  def test_routing!
+    any_up_state_changed = false
+    any_routing_state_changed = false
+    messages = []
+    all_default_route_uplinks_down = false
+    commands = []
+
+    @uplinks.each do |uplink|
+      up_state_changed, routing_state_changed, message = uplink.test_routing!
+      any_up_state_changed ||= up_state_changed
+      any_routing_state_changed ||= routing_state_changed
+      messages << message
+    end
+
+    default_route_uplinks = @uplinks.find_all { |uplink| uplink.default_route }
+    if default_route_uplinks.all? { |uplink| !uplink.up }
+      default_route_uplinks.each { |uplink| uplink.routing = true }
+      puts 'No default route uplink seems to be up: enabling them all!' if DEBUG
+      all_default_route_uplinks_down = true
+    end
+
+    #change default route if any uplink changed its routing state
+    if any_routing_state_changed
+      commands = set_default_route_commands
+      #apply the routing changes
+      commands += ['ip route flush cache']
+    end
+
+    messages = [] unless any_up_state_changed
+
+    [commands, messages, all_default_route_uplinks_down]
+  end
+
+end

data/lib/fault_tolerant_router/version.rb

@@ -1,3 +1,3 @@
 module FaultTolerantRouter
-  VERSION = '1.0.1'
+  VERSION = '1.1.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fault_tolerant_router
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Alessandro Zarrilli
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-10 00:00:00.000000000 Z
+date: 2015-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -79,6 +79,8 @@ files:
 - lib/fault_tolerant_router/generate_config.rb
 - lib/fault_tolerant_router/generate_iptables.rb
 - lib/fault_tolerant_router/monitor.rb
+- lib/fault_tolerant_router/uplink.rb
+- lib/fault_tolerant_router/uplinks.rb
 - lib/fault_tolerant_router/version.rb
 homepage: https://github.com/drsound/fault_tolerant_router
 licenses: