stellar-core-backup 0.0.5 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 036788c4ca590787bf980fb86d5a766f8ffc0cff
-  data.tar.gz: 2912860cd046d4fb50cae3572b9af9b5c8668da3
+  metadata.gz: 3c708f769686ce12524217fe911479cfe10aab8f
+  data.tar.gz: 9ffb182e1def72ef33cd6fba0b6e9c68289cd47b
 SHA512:
-  metadata.gz: d0225b353f3638ea8ba37de50e37e00ac710aff9541384f8bbf881b30bb260e3fa1007f90ebd9b3e18816ca3498b399882a3c83603b40f8782954be8dfcee581
-  data.tar.gz: 3cca8e3f4456cfc256e99951b612933bd0d572dc0756f14a75c66b58b6bbbf740d1e2657e2bd71ddd19940665b21f3f5ae9c2b4f2ec532ae38c2b971806dc979
+  metadata.gz: 49df44ceefb82383d46c86cf2b70275ce029990109f6fdff67a595218f970d5794d922f4e680c9c3d70d63a7594d53e6091e351482149d351bc0948d420a2af1
+  data.tar.gz: 60abb9dbbac3ae53955cdf96f5006c6b71d98c037e138f8f4586c58d6652f30d33c764221d67fcb20344abaec541d01431c446009a4cbe826936a9b8ca5c5bc5

data/README.md

@@ -29,6 +29,7 @@ We are using SKS Keyservers for public key distribution available at hkp://pool.
 |s3_bucket| S3 bucket to store/retrieve buckets to/from|
 |s3_path| S3 Path prefix, can be used for backing up multiple core nodes to the same bucket|
 |gpg_key| GPG key ID used for signing and verification of the stellar-core backups. The provided ID is the Stellar public key|
+|pushgateway_url| Optional prometheus pushgateway URL to publish metrics to|
 
 ## Usage As Command Line Tool
 

data/config/sample.yaml

@@ -5,3 +5,4 @@ s3_region: us-east-1
 s3_bucket: stellar002-core-backups
 s3_path: testnet
 gpg_key: 'D99E1D15ACC358AB'
+pushgateway_url: http://127.0.0.1:9091/metrics/job/stellar-core-backup/instance/dev

data/lib/stellar-core-backup/job.rb

@@ -17,23 +17,26 @@ module StellarCoreBackup
       @listlen  = args[:listlen]
 
       # Set common run time parameters
-      @job_type     = args[:type]
-      @gpg_key      = @config.get('gpg_key')
-      @working_dir  = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
-      @cmd          = StellarCoreBackup::Cmd.new(@working_dir)
-      @select       = args[:select] if args.has_key?(:select)
-      @s3           = StellarCoreBackup::S3.new(@config)
+      @job_type        = args[:type]
+      @gpg_key         = @config.get('gpg_key')
+      @working_dir     = StellarCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @cmd             = StellarCoreBackup::Cmd.new(@working_dir)
+      @select          = args[:select] if args.has_key?(:select)
+      @s3              = StellarCoreBackup::S3.new(@config)
+      @pushgateway_url = @config.get('pushgateway_url')
 
       # Set per operation type run time parameters
       if args.has_key?(:type) then
         case args[:type]
           when 'backup'
             puts 'info: backing up stellar-core'
+            StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_start_time')
             @backup_dir   = StellarCoreBackup::Utils.create_backup_dir(@config.get('backup_dir'))
             @db           = StellarCoreBackup::Database.new(@config)
             @fs           = StellarCoreBackup::Filesystem.new(@config)
           when 'restore'
             puts 'info: restoring stellar-core'
+            StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_start_time')
             @db_restore   = StellarCoreBackup::Restore::Database.new(@config)
             @fs_restore   = StellarCoreBackup::Restore::Filesystem.new(@config)
             @utils        = StellarCoreBackup::Utils.new(@config)
@@ -70,14 +73,23 @@ module StellarCoreBackup
           end
         when 'backup'
           begin
+            if !StellarCoreBackup::Utils.core_healthy?(@config) then
+                puts "error: Can't back up unhealthy stellar-core"
+                raise StandardError
+            end
             puts 'info: stopping stellar-core'
             # using sudo, if running as non root uid then you will need to configure sudoers
             stop_core = @cmd.run_and_capture('sudo', ['/bin/systemctl', 'stop', 'stellar-core'])
             # only proceed if core is stopped
             if stop_core.success then
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_db_dump_start_time')
               @db.backup
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_db_dump_finish_time')
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_fs_backup_start_time')
               @fs.backup
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_fs_backup_finish_time')
               if @verify then
+                StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_verify_start_time')
                 create_hash_file = @cmd.run_and_capture('find', ['.', '-type', 'f', '!', '-name', 'SHA256SUMS', '|', 'xargs', 'sha256sum', '>', 'SHA256SUMS'])
                 if create_hash_file.success then
                   puts "info: sha sums file created"
@@ -88,14 +100,19 @@ module StellarCoreBackup
                 sign_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--detach-sign', 'SHA256SUMS'])
                 if sign_hash_file.success then
                   puts "info: gpg signature created ok"
+                  StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_verify_finish_time')
                 else
                   puts 'error: error signing sha256sum file'
                   raise StandardError
                 end
               end
               # create tar archive with fs, db backup files and if requested the file of shas256sums and corresponding gpg signature.
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_tar_start_time')
               @backup = StellarCoreBackup::Utils.create_backup_tar(@working_dir, @backup_dir)
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_tar_finish_time')
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_s3_push_start_time')
               @s3.push(@backup)
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_s3_push_finish_time')
             else
               puts 'error: can not stop stellar-core'
               raise StandardError
@@ -118,6 +135,9 @@ module StellarCoreBackup
             puts e
             # clean up working_dir
             StellarCoreBackup::Utils.cleanup(@working_dir)
+            StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_fail_time')
+          else
+            StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_backup_success_time')
           end
         when 'restore'
           begin
@@ -133,9 +153,14 @@ module StellarCoreBackup
               # if no manual selection has been made, use the latest as derived from the s3.latest method
               # this method returns an array so set @select to the first and only element
               @select=@s3.latest(1)[0] if ! @select
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_s3_get_start_time')
               @backup_archive = @s3.get(@select)
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_s3_get_finish_time')
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_untar_start_time')
               @utils.extract_backup(@backup_archive)
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_untar_finish_time')
               if @verify then
+                StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_verify_start_time')
                 verify_hash_file = @cmd.run_and_capture('gpg', ['--local-user', @gpg_key, '--verify', 'SHA256SUMS.sig', 'SHA256SUMS', '2>&1'])
                 if verify_hash_file.success then
                   puts "info: gpg signature processed ok"
@@ -152,14 +177,19 @@ module StellarCoreBackup
                 end
                 if StellarCoreBackup::Utils.confirm_shasums_definitive(@working_dir, @backup_archive) then
                   puts 'info: SHA256SUMS file list matches delivered archive'
+                  StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_verify_finish_time')
                 else
                   puts 'error: unknown additional file(s) detected in archive'
                   raise StandardError
                 end
               end
               StellarCoreBackup::Utils.cleanbucket(@fs_restore.core_data_dir) if @clean
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_fs_restore_start_time')
               @fs_restore.restore(@backup_archive)
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_fs_restore_finish_time')
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_db_restore_start_time')
               @db_restore.restore()
+              StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_db_restore_finish_time')
 
               # restart stellar-core post restore
               puts 'info: starting stellar-core'
@@ -182,6 +212,9 @@ module StellarCoreBackup
             puts e
             # clean up working_dir
             StellarCoreBackup::Utils.cleanup(@working_dir)
+            StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_fail_time')
+          else
+            StellarCoreBackup::Utils.push_metric(@pushgateway_url, 'stellar_core_restore_success_time')
           end
       end
     end

data/lib/stellar-core-backup/utils.rb

@@ -1,4 +1,8 @@
 require 'fileutils'
+require 'net/http'
+require 'json'
+require 'time'
+
 
 module StellarCoreBackup
   class Utils
@@ -133,5 +137,61 @@ module StellarCoreBackup
       return Etc.nprocessors
     end
 
+    # check stellar-core status
+    Contract StellarCoreBackup::Config => Bool
+    def self.core_healthy?(config)
+      port = get_admin_port(config.get('core_config'))
+      url = "http://127.0.0.1:%s/info" % port
+      uri = URI(url)
+      begin
+        response = Net::HTTP.get(uri)
+        state = JSON.parse(response)['info']['state']
+        if state == 'Synced!' then
+          puts "info: stellar-core up and synced"
+          return true
+        else
+          puts "error: stellar-core status is: %s" % state
+          return false
+        end
+      rescue
+        puts "info: stellar-core down or not synced"
+        return false
+      end
+    end
+
+    private
+    Contract String => String
+    def self.get_admin_port(config)
+      File.open(config,'r') do |fd|
+        fd.each_line do |line|
+          if (line[/^HTTP_PORT=/]) then
+            port = /^HTTP_PORT=(.*)/.match(line).captures[0]
+            return port
+          end
+        end
+      end
+    end
+
+    # Publishes metric to pushgateway
+    # if no value provided current epoch timestamp will be used
+    Contract Any, String, Any => Bool
+    def self.push_metric(url, metric, value=nil)
+      if url.nil? then
+        # No url means pushgateway URL is not configured
+        return false
+      end
+
+      if value.nil? then
+          value = Time.now.to_i
+      end
+
+      uri = URI(url)
+      req = Net::HTTP::Post.new(uri.request_uri)
+      req.body = "%s %i\n" % [metric, value]
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.request(req)
+      return true
+    end
+
   end
 end

data/lib/stellar-core-backup/version.rb

@@ -1,3 +1,3 @@
 module StellarCoreBackup
-  VERSION = "0.0.5"
+  VERSION = "0.0.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stellar-core-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
 platform: ruby
 authors:
 - Tom Llewellyn-Smith
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-22 00:00:00.000000000 Z
+date: 2021-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: contracts