digitalbits-core-backup 0.0.7

data/lib/digitalbits-core-backup/utils.rb

@@ -0,0 +1,197 @@
+require 'fileutils'
+require 'net/http'
+require 'json'
+require 'time'
+
+
+module DigitalbitsCoreBackup
+  class Utils
+    include Contracts
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @db_restore   = DigitalbitsCoreBackup::Restore::Database.new(@config)
+      @fs_restore   = DigitalbitsCoreBackup::Restore::Filesystem.new(@config)
+    end
+
+    Contract String => String
+    def self.create_working_dir(dir)
+      working_dir = dir + "/#{Process.pid}"
+      unless Dir.exists?(working_dir) then
+        Dir.mkdir working_dir
+      end
+      return working_dir
+    end
+
+    Contract String => nil
+    def self.remove_working_dir(working_dir)
+      if Dir.exists?(working_dir) then
+        Dir.rmdir working_dir + "/#{Process.pid}"
+      end
+    end
+
+    Contract String => String
+    def self.create_backup_dir(dir)
+      unless Dir.exists?(dir) then
+        Dir.mkdir dir
+      end
+      return dir
+    end
+
+    Contract String, String => String
+    def self.create_backup_tar(working_dir, backup_dir)
+      puts 'info: creating backup tarball'
+      tar_file = "#{backup_dir}/core-backup-#{Time.now.to_i}.tar"
+      Dir.chdir(working_dir)
+      # archive the working directory
+      DigitalbitsCoreBackup::Tar.pack(tar_file, '.')
+      return tar_file
+    end
+
+    Contract String => nil
+    def extract_backup(backup_archive)
+      # extract the backup archive into the working directory
+      DigitalbitsCoreBackup::Tar.unpack(backup_archive, @working_dir)
+      return
+    end
+
+#    Contract String => nil
+#    def restore(backup_archive)
+#      @fs_restore.restore(backup_archive)
+#      @db_restore.restore()
+#    end
+
+    Contract String => Bool
+    def self.cleanbucket(bucket_dir)
+      if FileUtils.remove(Dir.glob(bucket_dir+'/*')) then
+        puts 'info: cleaning up workspace'
+        return true
+      else
+        return false
+      end
+    end
+
+    Contract String => Bool
+    def self.cleanup(working_dir)
+      if FileUtils.remove_dir(working_dir) then
+        puts 'info: cleaning up workspace'
+        return true
+      else
+        return false
+      end
+    end
+
+    Contract String, String => Bool
+    def self.confirm_shasums_definitive(working_dir, backup_archive)
+
+      # create an array of filesunpacked into the working_dir
+      Dir.chdir(working_dir)
+      files_present=Dir.glob('./**/*')
+
+      # remove directories and shasum details from file array
+      files_present.delete('./'+File.basename(backup_archive))
+      files_present.delete('./core-db')
+      files_present.delete('./SHA256SUMS')
+      files_present.delete('./SHA256SUMS.sig')
+
+      # now delete the file names in the shasums file from the array
+      # we are expecting an array of zero length after this process
+      File.open("SHA256SUMS").each { |sha_file| files_present.delete(sha_file.split(' ')[1].chomp) }
+      if files_present.none? then
+        return true
+      else
+        return false
+      end
+    end
+
+    # check we have read permissions
+    Contract String => Bool
+    def self.readable?(file)
+      if File.readable?(file) then
+        puts "info: #{file} readable"
+        return true
+      else
+        puts "error: cannot read #{file}"
+        raise Errno::EACCES
+      end
+    end
+
+    # check we have write permissions
+    Contract String => Bool
+    def self.writable?(file)
+      if File.writable?(file) then
+        puts "info: #{file} writeable"
+        return true
+      else
+        puts "error: cannot write to #{file}"
+        raise Errno::EACCES
+      end
+    end
+
+    # return number of available cores
+    Contract nil => Integer
+    def self.num_cores?()
+      require 'etc'
+      return Etc.nprocessors
+    end
+
+    # check digitalbits-core status
+    Contract DigitalbitsCoreBackup::Config => Bool
+    def self.core_healthy?(config)
+      port = get_admin_port(config.get('core_config'))
+      url = "http://127.0.0.1:%s/info" % port
+      uri = URI(url)
+      begin
+        response = Net::HTTP.get(uri)
+        state = JSON.parse(response)['info']['state']
+        if state == 'Synced!' then
+          puts "info: digitalbits-core up and synced"
+          return true
+        else
+          puts "error: digitalbits-core status is: %s" % state
+          return false
+        end
+      rescue
+        puts "info: digitalbits-core down or not synced"
+        return false
+      end
+    end
+
+    private
+    Contract String => String
+    def self.get_admin_port(config)
+      File.open(config,'r') do |fd|
+        fd.each_line do |line|
+          if (line[/^HTTP_PORT=/]) then
+            port = /^HTTP_PORT=(.*)/.match(line).captures[0]
+            return port
+          end
+        end
+      end
+    end
+
+    # Publishes metric to pushgateway
+    # if no value provided current epoch timestamp will be used
+    Contract Any, String, Any => Bool
+    def self.push_metric(url, metric, value=nil)
+      if url.nil? then
+        # No url means pushgateway URL is not configured
+        return false
+      end
+
+      if value.nil? then
+          value = Time.now.to_i
+      end
+
+      uri = URI(url)
+      req = Net::HTTP::Post.new(uri.request_uri)
+      req.body = "%s %i\n" % [metric, value]
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.request(req)
+      return true
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/utils.rb-e

@@ -0,0 +1,197 @@
+require 'fileutils'
+require 'net/http'
+require 'json'
+require 'time'
+
+
+module DigitalbitsCoreBackup
+  class Utils
+    include Contracts
+
+    Contract DigitalbitsCoreBackup::Config => Contracts::Any
+    def initialize(config)
+      @config       = config
+      @working_dir  = DigitalbitsCoreBackup::Utils.create_working_dir(@config.get('working_dir'))
+      @db_restore   = DigitalbitsCoreBackup::Restore::Database.new(@config)
+      @fs_restore   = DigitalbitsCoreBackup::Restore::Filesystem.new(@config)
+    end
+
+    Contract String => String
+    def self.create_working_dir(dir)
+      working_dir = dir + "/#{Process.pid}"
+      unless Dir.exists?(working_dir) then
+        Dir.mkdir working_dir
+      end
+      return working_dir
+    end
+
+    Contract String => nil
+    def self.remove_working_dir(working_dir)
+      if Dir.exists?(working_dir) then
+        Dir.rmdir working_dir + "/#{Process.pid}"
+      end
+    end
+
+    Contract String => String
+    def self.create_backup_dir(dir)
+      unless Dir.exists?(dir) then
+        Dir.mkdir dir
+      end
+      return dir
+    end
+
+    Contract String, String => String
+    def self.create_backup_tar(working_dir, backup_dir)
+      puts 'info: creating backup tarball'
+      tar_file = "#{backup_dir}/core-backup-#{Time.now.to_i}.tar"
+      Dir.chdir(working_dir)
+      # archive the working directory
+      DigitalbitsCoreBackup::Tar.pack(tar_file, '.')
+      return tar_file
+    end
+
+    Contract String => nil
+    def extract_backup(backup_archive)
+      # extract the backup archive into the working directory
+      DigitalbitsCoreBackup::Tar.unpack(backup_archive, @working_dir)
+      return
+    end
+
+#    Contract String => nil
+#    def restore(backup_archive)
+#      @fs_restore.restore(backup_archive)
+#      @db_restore.restore()
+#    end
+
+    Contract String => Bool
+    def self.cleanbucket(bucket_dir)
+      if FileUtils.remove(Dir.glob(bucket_dir+'/*')) then
+        puts 'info: cleaning up workspace'
+        return true
+      else
+        return false
+      end
+    end
+
+    Contract String => Bool
+    def self.cleanup(working_dir)
+      if FileUtils.remove_dir(working_dir) then
+        puts 'info: cleaning up workspace'
+        return true
+      else
+        return false
+      end
+    end
+
+    Contract String, String => Bool
+    def self.confirm_shasums_definitive(working_dir, backup_archive)
+
+      # create an array of filesunpacked into the working_dir
+      Dir.chdir(working_dir)
+      files_present=Dir.glob('./**/*')
+
+      # remove directories and shasum details from file array
+      files_present.delete('./'+File.basename(backup_archive))
+      files_present.delete('./core-db')
+      files_present.delete('./SHA256SUMS')
+      files_present.delete('./SHA256SUMS.sig')
+
+      # now delete the file names in the shasums file from the array
+      # we are expecting an array of zero length after this process
+      File.open("SHA256SUMS").each { |sha_file| files_present.delete(sha_file.split(' ')[1].chomp) }
+      if files_present.none? then
+        return true
+      else
+        return false
+      end
+    end
+
+    # check we have read permissions
+    Contract String => Bool
+    def self.readable?(file)
+      if File.readable?(file) then
+        puts "info: #{file} readable"
+        return true
+      else
+        puts "error: cannot read #{file}"
+        raise Errno::EACCES
+      end
+    end
+
+    # check we have write permissions
+    Contract String => Bool
+    def self.writable?(file)
+      if File.writable?(file) then
+        puts "info: #{file} writeable"
+        return true
+      else
+        puts "error: cannot write to #{file}"
+        raise Errno::EACCES
+      end
+    end
+
+    # return number of available cores
+    Contract nil => Integer
+    def self.num_cores?()
+      require 'etc'
+      return Etc.nprocessors
+    end
+
+    # check digitalbits-core status
+    Contract DigitalbitsCoreBackup::Config => Bool
+    def self.core_healthy?(config)
+      port = get_admin_port(config.get('core_config'))
+      url = "http://127.0.0.1:%s/info" % port
+      uri = URI(url)
+      begin
+        response = Net::HTTP.get(uri)
+        state = JSON.parse(response)['info']['state']
+        if state == 'Synced!' then
+          puts "info: digitalbits-core up and synced"
+          return true
+        else
+          puts "error: digitalbits-core status is: %s" % state
+          return false
+        end
+      rescue
+        puts "info: digitalbits-core down or not synced"
+        return false
+      end
+    end
+
+    private
+    Contract String => String
+    def self.get_admin_port(config)
+      File.open(config,'r') do |fd|
+        fd.each_line do |line|
+          if (line[/^HTTP_PORT=/]) then
+            port = /^HTTP_PORT=(.*)/.match(line).captures[0]
+            return port
+          end
+        end
+      end
+    end
+
+    # Publishes metric to pushgateway
+    # if no value provided current epoch timestamp will be used
+    Contract Any, String, Any => Bool
+    def self.push_metric(url, metric, value=nil)
+      if url.nil? then
+        # No url means pushgateway URL is not configured
+        return false
+      end
+
+      if value.nil? then
+          value = Time.now.to_i
+      end
+
+      uri = URI(url)
+      req = Net::HTTP::Post.new(uri.request_uri)
+      req.body = "%s %i\n" % [metric, value]
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.request(req)
+      return true
+    end
+
+  end
+end

data/lib/digitalbits-core-backup/version.rb

@@ -0,0 +1,3 @@
+module DigitalbitsCoreBackup
+  VERSION = "0.0.7"
+end

data/lib/digitalbits-core-backup/version.rb-e

@@ -0,0 +1,3 @@
+module DigitalbitsCoreBackup
+  VERSION = "0.0.7"
+end

data/lib/digitalbits-core-backup.rb

@@ -0,0 +1,21 @@
+require "digitalbits-core-backup/version"
+require "contracts"
+require "fileutils"
+require "pg"
+
+module DigitalbitsCoreBackup
+  autoload :Cmd, "digitalbits-core-backup/cmd"
+  autoload :CmdResult, "digitalbits-core-backup/cmd_result"
+  autoload :Config, "digitalbits-core-backup/config"
+  autoload :Database, "digitalbits-core-backup/database"
+  autoload :Filesystem, "digitalbits-core-backup/filesystem"
+  autoload :Job, "digitalbits-core-backup/job"
+  autoload :S3, "digitalbits-core-backup/s3"
+  autoload :Tar, "digitalbits-core-backup/tar"
+  autoload :Utils, "digitalbits-core-backup/utils"
+
+  module Restore
+    autoload :Database, "digitalbits-core-backup/restore/database"
+    autoload :Filesystem, "digitalbits-core-backup/restore/filesystem"
+  end
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: digitalbits-core-backup
+version: !ruby/object:Gem::Version
+  version: 0.0.7
+platform: ruby
+authors:
+- Digitalbits
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2022-10-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: contracts
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- support@digitalbits.io
+executables:
+- digitalbits-core-backup
+extensions: []
+extra_rdoc_files: []
+files:
+- CONTRIBUTING.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/digitalbits-core-backup
+- config/sample.yaml
+- digitalbits-core-backup.gemspec
+- lib/digitalbits-core-backup.rb
+- lib/digitalbits-core-backup/cmd.rb
+- lib/digitalbits-core-backup/cmd.rb-e
+- lib/digitalbits-core-backup/cmd_result.rb
+- lib/digitalbits-core-backup/cmd_result.rb-e
+- lib/digitalbits-core-backup/config.rb
+- lib/digitalbits-core-backup/config.rb-e
+- lib/digitalbits-core-backup/database.rb
+- lib/digitalbits-core-backup/database.rb-e
+- lib/digitalbits-core-backup/filesystem.rb
+- lib/digitalbits-core-backup/filesystem.rb-e
+- lib/digitalbits-core-backup/job.rb
+- lib/digitalbits-core-backup/job.rb-e
+- lib/digitalbits-core-backup/restore/database.rb
+- lib/digitalbits-core-backup/restore/database.rb-e
+- lib/digitalbits-core-backup/restore/filesystem.rb
+- lib/digitalbits-core-backup/restore/filesystem.rb-e
+- lib/digitalbits-core-backup/s3.rb
+- lib/digitalbits-core-backup/s3.rb-e
+- lib/digitalbits-core-backup/tar.rb
+- lib/digitalbits-core-backup/tar.rb-e
+- lib/digitalbits-core-backup/utils.rb
+- lib/digitalbits-core-backup/utils.rb-e
+- lib/digitalbits-core-backup/version.rb
+- lib/digitalbits-core-backup/version.rb-e
+homepage: https://github.com/xdbfoundation/digitalbits-core-backup
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: A helper script to backup a digitalbits-core node
+test_files: []