aws_site_monitor 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1b4efdad1bd774afcda2cdfb73079a02d503ee84
-  data.tar.gz: 7f076bb696837c26bf682705dba6cf1723eedc1d
+  metadata.gz: e84a55fbc38e5cd50eafc7d9356957c906322587
+  data.tar.gz: e01a652c8a7e4bdb68b789649d25f69125d74dae
 SHA512:
-  metadata.gz: ce4dbcc315b7cc39fce9afb36596774752162f5dbb9f543367941b9e5c14ceca25cccd1dd19342efa5ef7eeb4a8cd8140cd68e681739ad91b6878a892bd4eb64
-  data.tar.gz: 4b768c6daa6ed908d4d6582b4a090da4d61b115664d4902f27aeb2af1bd70b7524083ef87721771d62b75441c84caa964843430f775c322b6cc4f060fc86e47f
+  metadata.gz: a4582e145791b09af30ce1679e2cc51829029c669e4729c403db7473c7659d157a538ab5aa839b675807bacfdbf0ca026eea0054f7c9fc3e44c241a2c6df7062
+  data.tar.gz: bf2db8bc432d4d8923cce891a45aa84e5ea90f4e2da7fd2e41afbe056d248347d5a1a7c0bd80619c6d49e73a14a7760acf420fe33776259aaa9d16a9b1af6156

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    aws_site_monitor (0.1.2)
+    aws_site_monitor (0.1.3)
       activesupport
       aws-sdk
       concurrent-ruby

data/lib/aws/site_monitor/cli.rb

@@ -5,6 +5,7 @@ module Aws
       option :aws_region, :type => :string, :default => 'us-east-1', :desc => 'AWS region'
       option :killswitch_url, :type => :string, :desc => 'If a file no longer exists at this url, kill script'
       option :request_timeout_seconds, :type => :numeric, :default => 15, :desc => 'How long to wait for response before request times out which will trigger a reboot'
+      option :attempts_until_hard_stop, :type => :numeric, :default => 0, :desc => 'How many reboot attempts before attempting hard stop, 0 = disabled'
 
       desc "start", "Start Watching"
       def start
@@ -83,9 +84,11 @@ module Aws
 
               if result[0] == "2"
                 puts "GOT 200 EVERYTHING OK"
+                site.reset_failure_count
               else
+                site.track_failure
                 ::Aws::SiteMonitor::Event.create(:status_code => result)
-                site.reboot_instances!
+                site.handle_failure!(options)
               end
             end
           end

data/lib/aws/site_monitor/site.rb

@@ -3,6 +3,39 @@ module Aws
     class Site
       include ::Aws::SiteMonitor::PstoreRecord
 
+      def initialize(**options)
+        super(**options)
+        @failure_count ||= 0
+      end
+
+      #this handles issues where instance cannot be restarted due to normal reboot.
+      #since hard stopping takes longer, we don't want to do it unless necessary.
+      #the reset failure count is semi wonky because it will reset even if success
+      #hasnt yet been reached, however it is much less complicated than trying to
+      #query state of multiple instances. basically if threshold is reached, hard stop
+      #the instances, then on next pass since we reset the failure count, it should attempt
+      #normal reboot, which will cause the instances to start
+      def handle_failure!(options)
+        hard_stop_enabled = options.attempts_until_hard_stop > 0
+
+        if hard_stop_enabled && self[:failure_count] == options.attempts_until_hard_stop
+          reset_failure_count
+          stop_instances!
+        else
+          reboot_instances!
+        end
+      end
+
+      def track_failure
+        @failure_count = @failure_count + 1
+        save
+      end
+
+      def reset_failure_count
+        @failure_count = 0
+        save
+      end
+
       def reboot_instances!
         puts "RESTARTING SITE #{self.attributes}"
         ::Aws::SiteMonitor.ec2_client.reboot_instances(
@@ -19,6 +52,14 @@ module Aws
           instance_ids: self[:instance_ids]
         )
       end
+
+      def stop_instances!
+        ::Aws::SiteMonitor.ec2_client.stop_instances(
+          instance_ids: self[:instance_ids]
+        )
+      rescue ::Aws::EC2::Errors::IncorrectState => e
+        start_instances!
+      end
     end
   end
 end

data/lib/aws/site_monitor/version.rb

@@ -1,5 +1,5 @@
 module Aws
   module SiteMonitor
-    VERSION = "0.1.2"
+    VERSION = "0.1.3"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws_site_monitor
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Jason Ayre
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-29 00:00:00.000000000 Z
+date: 2018-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler