nixadm 1.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c7d91b74d9982ad275f6432d2ed60e6835a3e2bc05e057da200c927200c00998
+  data.tar.gz: 2873bd8d7aed1583f8bb7464a8114911f4886af8b79024336b7c99d978177ebb
+SHA512:
+  metadata.gz: f2055495232a0a6d87f9fa7174daead46b7148606d633677e06b046c20237f2f43951510b085169b9e5cb668a6c8fef0ed850cbafccc3775a481ec4df5346d3a
+  data.tar.gz: ac9bef4a7749c7f812b0fbea27965bd1a0e5209120d56b6ca1ea9629585e84cbaf84b28eeffbbcc8f75dee47472be18c85943f40eec0703a37c7456679e9d18a

data/README.md

@@ -0,0 +1,26 @@
+# NixAdm
+
+## About
+
+This is a general library designed to aid with system administration tasks,
+specifically to replace all shell scripting with Ruby code.
+
+## Installation
+
+You can install directly from the command line via Ruby gems as follows:
+
+    gem install nixadm
+
+## Building from Source
+
+This project uses CMake. To build the gem:
+
+    cmake .
+    make gem
+
+The resultant gem will be generated in the <tt>pkg</tt> directory.
+
+## License
+
+Copyright information is located in the COPYING file. The software license is
+located in the LICENSE file.

data/src/lib/nixadm/backup.rb

@@ -0,0 +1,267 @@
+require 'nixadm/util'
+require 'nixadm/zfs'
+
+module NixAdm
+
+# Base backup class. Provides centralized logging facility and job start/stop
+# templates.
+class Backup < NixAdm::Command
+
+  attr_reader :host
+
+  def initialize(host, options = {})
+    super(host, options)
+
+    if not $backup_dir.nil?
+      @logfile = File.open("#{$backup_dir}/backup.log", 'w')
+    end
+  end
+
+  def run()
+    startBackup()
+    yield
+    endBackup()
+  end
+
+  def startBackup()
+
+  end
+
+  def endBackup()
+
+  end
+
+  #------------------------------------------------------------------------------
+  # Utility functions
+  #------------------------------------------------------------------------------
+
+  protected
+
+  def cd(path)
+    Dir.chdir(path)
+  end
+
+  def replicationStatus(mfs, sfs)
+    h1 = mfs.pool.host.name
+    p1 = mfs.pool.name
+    f1 = mfs.name
+    h2 = sfs.pool.host.name
+    p2 = sfs.pool.name
+    f2 = sfs.name
+
+    return "replicating #{h1}:#{p1}/#{f1} -> #{h2}:#{p2}/#{f2}"
+  end
+
+  # Run general shell command and log output
+  def run_command(command)
+    sys = NixAdm::Pipeline.new()
+
+    sys.classic(command) do |stdin, stdout|
+      stdin.close()
+
+      while true
+        line = stdout.gets
+        break if line.nil?
+        @logfile.write(line)
+      end
+    end
+  end
+
+  # Executes a command, captures output into file and zips the file
+  # @param command A command or array containing a pipeline of commands
+  # @param filename The name (path) of the file to dump database to
+  def zipStore(host, command, filename)
+    store host, command, filename, zip: true
+  end
+
+  # Executes a command, captures output into file
+  # @param command A command or array containing a pipeline of commands
+  # @param filename The name (path) of the file to dump database to
+  # @param zip If true, gzip the file.
+  def store(host, command, filename, zip: false)
+    sys      = NixAdm::Pipeline.new()
+    pipeline = nil
+
+    sys.debug = debug()
+
+    # If the host we are performing this command on is the same host, don't
+    # resolve with host as secong argument, as this just makes that host ssh
+    # back into itself to run the command.
+    if @host == host
+      pipeline = resolveCommand(command)
+    else
+      pipeline = resolveCommand(command, host)
+    end
+
+    sys.run([pipeline]) do |stdin, stdout|
+      stdin.close()
+
+      dumpfile = File.open(filename, 'wb')
+
+      while true
+        data = stdout.read(1024)
+        break if data == nil
+        dumpfile.write(data)
+      end
+
+      dumpfile.close()
+    end
+
+    if zip == true
+      # Compress the file
+      sys.run "gzip -f #{filename}"
+    end
+  end
+
+  def backup_dir(dir)
+    return "#{$backup_dir}/#{dir}"
+  end
+
+  # Replicates ZFS filesystem(s) on from host to backup hosts
+  #
+  # @param host The main host we are backing up
+  # @param filesystems An array of ZFS filesystems to replicate
+  #
+  # @param targets An array or hashes defining target hosts to replicate
+  # to. It has the following form:
+  #
+  #  targets = [ { host: 'root@midway',  root: 'pool' },
+  #              { host: 'root@backup.c21bowman.com',  root: 'pool' },
+  #              { host: 'root@arc2', root: 'pool' } ]
+  #
+  def replicate(host, objects, targets)
+    admin = NixAdm::ZFS::Admin.new(host)
+
+    objects.each do |object|
+
+      # First look for matching filesystem
+      master_object = admin.object(object)
+
+      if master_object.nil?
+        raise "Error: #{object} does not exist"
+      end
+
+      startSourceZfsReplication master_object
+
+      failed = false
+      targets.each do | target |
+
+        if target[:host].nil? or target[:pool].nil?
+          raise "Invalid arguments: missing host or pool"
+        end
+
+        slave = NixAdm::ZFS::Host.new(target[:host])
+        pool  = slave.pool(target[:pool])
+
+        attempts = 0
+        while attempts < 2
+          slave_object = pool.object(master_object.name)
+
+          break if not slave_object.nil?
+
+          # If we get here, it means the remote object does not exists. If this
+          # is a filesystem, we must first create it. If it's a zvol, we do
+          # nothing.
+
+          $stderr.puts "Dest ZFS entity '#{pool.name}/#{master_object.name}' " +
+            "does not exist ... creating"
+
+          if master_object.is_a?(NixAdm::ZFS::Filesystem)
+            if pool.createFilesystem(master_object.name) == false
+              msg = "Error: ZFS entity '#{pool.name}/#{master_object.name}' " + \
+              "does not exist and could not be created"
+              raise msg
+            end
+          elsif master_object.is_a?(NixAdm::ZFS::Volume)
+            if pool.createVolume(master_object.name) == false
+              msg = "Error: ZFS entity '#{pool.name}/#{master_object.name}' " + \
+              "does not exist and could not be created"
+              raise msg
+            end
+          end
+
+          attempts += 1
+        end
+
+        if block_given?
+          yield master_object, slave_object
+        end
+
+        if admin.replicate(master_object, slave_object) == true
+          endDestZfsReplication slave_object
+        else
+          failed = true
+          msg    = "FAILED: #{job} - #{admin.status.msg}"
+          log msg, admin.status.val
+        end
+      end
+
+      if failed == false
+        endSourceZfsReplication master_object
+      end
+    end
+  end
+
+  def startSourceZfsReplication(master_object)
+    return success()
+  end
+
+  def endDestZfsReplication(slave_object)
+    return success()
+  end
+
+  def endSourceZfsReplication(master_object)
+    return success()
+  end
+
+  # Trim snaphots on given filesystem
+  #
+  # @param host The host name
+  # @param filesystems A list of filesystems to trim snapshots
+  def trim(host, objects)
+    admin = NixAdm::ZFS::Admin.new(host)
+
+    objects.each do |object|
+      master_object = admin.object(object)
+
+      if not master_object.nil?
+        master_object.trimSnapshots()
+      end
+    end
+  end
+
+end
+
+class PrimaryBackup < Backup
+
+  def initialize(host)
+    super
+  end
+
+  def startSourceZfsReplication(master_object)
+    return master_object.createSnapshot()
+  end
+
+  def endSourceZfsReplication(master_object)
+    return master_object.trimSnapshots()
+  end
+
+end # module PrimaryBackup
+
+class SecondaryBackup < Backup
+
+  def initialize(host)
+    super
+  end
+
+  def endDestZfsReplication(slave_object)
+    return slave_object.trimSnapshots()
+  end
+
+  def endSourceZfsReplication(master_object)
+    return master_object.trimSnapshots()
+  end
+
+end # module SecondaryBackup
+
+end # module NixAdm

data/src/lib/nixadm/db/postgres.rb

@@ -0,0 +1,9 @@
+require 'nixadm/util'
+require 'nixadm/backup'
+
+module NixAdm
+module PostreSQL
+
+
+end # module PostreSQL
+end # module NixAdm

data/src/lib/nixadm/db/postgresql.rb

@@ -0,0 +1,217 @@
+require 'nixadm/util'
+require 'nixadm/backup'
+
+module NixAdm
+module PostgreSQL
+
+module Replication
+
+class Node
+
+  include NixAdm::Util
+
+  def initialize(host, config_file=nil)
+    @host = host
+    @db   = connectDb(host, config_file)
+
+    if @db.nil?
+      raise "Connection failed to #{host}"
+    end
+
+    logSystemInit(config_file)
+    @logfields[:system] = 'psql'
+  end
+
+  def status()
+    return @db.exec('select * from pg_stat_replication')
+  end
+
+end # class Node
+
+# Primary database instance
+class Primary < Node
+
+  include NixAdm::Util
+
+  attr_reader :db
+
+  def initialize(host, config_file=nil)
+    super
+  end
+
+  def currentLsm()
+    return @db.exec('select pg_current_wal_lsn()')[0]
+  end
+
+end # class Master
+
+# Hot standby instance
+class Secondary < Node
+
+  include NixAdm::Util
+
+  attr_reader :primary
+
+  def initialize(primary, host, config_file=nil)
+    @primary = primary
+
+    super(host, config_file)
+  end
+
+  def currentLsm()
+    return @db.exec('select pg_last_wal_receive_lsn()')[0]
+  end
+
+end # class Replica
+
+end # module Replication
+
+# This class exports an entire PostgreSQL database cluster to $backup_dir/db,
+# including the relevant global settings.
+#
+# Example:
+#
+#   export = Export.new('db', { :debug => true })
+#   export.run()
+
+class Export < NixAdm::Backup
+
+  attr_accessor :filesystems, :targets
+
+  def initialize(host, options = {})
+    super host, options: options
+
+    @ssh_opts = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+  end
+
+  def log(msg, code=0)
+    $stderr.puts msg
+  end
+
+  #------------------------------------------------------------------------------
+  # Main backup function
+  #------------------------------------------------------------------------------
+
+  def run()
+    begin
+      backup_databases()
+    rescue Exception => ex
+      log "Backup failed: #{ex.to_s}\n #{ex.backtrace}"
+    end
+  end
+
+  #------------------------------------------------------------------------------
+  # Individual backup procedures
+  #------------------------------------------------------------------------------
+
+  # This does a pg_dump of all databases in plaintext format. It gzips them all.
+  def backup_databases()
+    cd backup_dir('db')
+
+    # List of databases on server
+    databases = listDatabases()
+
+    # List of databases to exclude
+    exclude =
+    {
+      'template0' => true,
+      'template1' => true
+    }
+
+    # Backup individual databases
+    databases.each do |db|
+      next if exclude.has_key?(db)
+
+      begin
+        log "Dumping: #{db}"
+        store @host, "pg_dump -p 5433 #{db}", "#{db}.db"
+      rescue
+        puts $!
+        log "Failed #{@host} #{db}: #{$?.to_s}", 1
+        next
+      end
+    end
+
+    # Dump the globals
+    globals_file = 'globals.sql'
+    log "Dumping globals to #{globals_file}"
+    store @host, 'pg_dumpall -g -p 5433', globals_file
+  end
+
+  protected
+
+  def listDatabases()
+    command = %Q{ psql -At -p 5433 postgres -c 'select datname from pg_database' }
+    pipeline = resolveCommand(command)
+
+    dbs = []
+    exec([pipeline]) do |line|
+      dbs << line.strip
+    end
+
+    return dbs
+  end
+
+end # class Export
+
+# This class imports everything exported from the Export class. Before
+# importing, you need to make sure you initialize your database with proper
+# encoding and collation. To do so, switch user to postgres. Then run:
+#
+#  /usr/local/bin/initdb --pgdata=/var/lib/pgsql9/data/ -E 'UTF-8' \
+#    --lc-collate='en_US.UTF-8' --lc-ctype='en_US.UTF-8';
+#
+#  /usr/local/bin/createuser -sl root
+#
+# Then start Postgres. Now you can import databases
+
+class Import < NixAdm::Backup
+
+  def initialize(host, options = {})
+    super host, options: options
+
+    @ssh_opts = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+  end
+
+  #------------------------------------------------------------------------------
+  # Main backup function
+  #------------------------------------------------------------------------------
+
+  def run()
+    begin
+      import_databases()
+    rescue Exception => ex
+      log "Import failed: #{ex.to_s}\n #{ex.backtrace}"
+    end
+  end
+
+  def import_databases()
+    dbs = Dir.entries($backup_dir)
+
+    # Restore globals first
+    system "psql -p 5433 -d postgres -f #{$backup_dir}/globals.sql"
+
+    dbs.each do |file|
+      next if File.extname(file) != '.db'
+
+      db = file.split('.')[0]
+      puts "Loading #{db}"
+
+      sql   = %q{ select count(*) from pg_database where datname='dba' }
+      count = %x{ psql -p 5433 postgres -At -c "#{sql}" }
+      count.strip!
+
+      if count == '1'
+        %x{ psql -p 5433 postgres -c 'drop database #{db}' }
+      end
+
+      %x{ psql -p 5433 postgres -c 'create database #{db}' }
+      %x{ psql -p 5433 -d #{db} -f #{$backup_dir}/#{file}  }
+      %x{ psql -p 5433 #{db}    -c 'analyze' }
+    end
+  end
+
+end # class Import
+
+end # module PostgreSQL
+end # module NixAdm