nixadm 1.0.6

data/src/lib/nixadm/util.rb

@@ -0,0 +1,210 @@
+require 'pg'
+require 'rbconfig'
+require 'securerandom'
+require 'yaml'
+require 'socket'
+
+require 'nixadm/version'
+
+#-------------------------------------------------------------------------------
+# Global parameters
+#-------------------------------------------------------------------------------
+
+# Configuration file. Config file is YAML.
+
+$nixadm_config_file = 'nixadm.conf'
+#-------------------------------------------------------------------------------
+
+class String
+
+  def sqlEscape()
+    return self.gsub("'", "''").gsub("\\", "\\\\")
+  end
+
+end
+
+# This extends PG::Result making it more "rubyish"
+module PG
+
+class Result
+
+  # Takes a row and column and return associated field. +r+ is the row
+  # ordinal. +c+ can be either a string (column name) or an integer
+  # (column ordinal).
+  def [](r, c)
+    idx = c
+
+    if c.class == String
+      idx = self.fnumber(c)
+    end
+
+    return self.getvalue(r, idx)
+  end
+
+  # This is meant to be used with +each()+. It takes a column and
+  # returns the associated field value. The current row is the active
+  # row in the +each()+ block. +c+ can be a String (column name) or an
+  # integer (column ordinal).
+  def [](c)
+    idx = c
+
+    if c.class == String
+      idx = self.fnumber(c)
+    end
+
+    return self.getvalue(@row || 0, idx)
+  end
+
+end # class PG
+end # module Result
+
+module NixAdm
+
+# Optional mixin for general utility functions
+
+module Util
+
+  # Run shell command in bash with pipefail option for full error detection on
+  # pipelines.
+  def bash(cmd)
+    cmd = "set -o pipefail && #{cmd}"
+    system("bash -c #{cmd.shellescape}")
+    cmd_status = $?
+    if cmd_status != 0
+      raise "'#{cmd}' execution failed (exit code: #{cmd_status})"
+    end
+  end
+
+  # Logging to PostgreSQL
+  #
+  # Connect to a database. Uses configuration file. Config file is YAML and
+  # logger uses the following format for database connection:
+  #
+  # ---
+  # databases:
+  #   logger:
+  #     host: db
+  #     db: nixadm
+  #     user: bob
+  #     password: bobspassword
+  #
+  # @param db_name The name of the entry in the configuration file for the
+  #   database connection paramaters.
+  #
+  # @return Database connection if successful, nil otherwise
+
+  def logSystemInit(config_file = nil)
+    @logdb = connectDb('logger', config_file)
+
+    @logfields =
+    {
+      :host     =>  Socket.gethostname,
+      :system   => 'backup',
+      :module   => self.class.name,
+      :function => 'run'
+    }
+  end
+
+  # Call when complete backup is done. Disconnects from database
+
+  def logSystemfinalize()
+    @logdb.disconnect()
+  end
+
+  # Connect to a database. Uses configuration file. Config file is YAML and has
+  # following format for databases:
+  #
+  # ---
+  # databases:
+  #   logger:
+  #     host: db
+  #     db: nixadm
+  #     user: bob
+  #     password: bobspassword
+  #
+  # @param db_name The name of the entry in the configuration file for the
+  #   database connection paramaters.
+  #
+  # @return Database connection if successful, nil otherwise
+
+  def connectDb(db_name, config_file = nil)
+
+    if config_file.nil?
+      config_file = $nixadm_config_file
+
+      # Account for BSD systems that user /usr/local/etc
+      prefix = RbConfig::CONFIG['prefix']
+      if prefix == '/usr/local'
+        config_file = prefix + $nixadm_config_file
+      end
+    end
+
+    @config = YAML::load_file(config_file)
+
+    params   = @config['databases'][db_name]
+
+    login    = params['user']
+    password = params['password']
+    db       = params['db']
+    host     = params['host']
+    port     = params['port'] || '5432'
+
+    return PG::Connection.new(host, port, '', '', db, login, password)
+  end
+
+  # Called before starting a backup job. Sets service function name based on
+  # called method and logs a start message.
+
+  def jobStart(host=nil)
+
+    if not host.nil?
+      @logfields[:function] = "#{caller_locations()[0].label} #{host} "
+    else
+      @logfields[:function] = "#{caller_locations()[0].label}"
+    end
+
+    @job = SecureRandom.uuid()
+
+    log('Started', 0)
+  end
+
+  # Call when job has completed. Logs a completed message and clears the service
+  # function name.
+
+  def jobFinish()
+    log('Completed', 0)
+    @logfields[:function] = ''
+  end
+
+  # Logs a message. Uses service name defined in jobStart().
+  #
+  # @param msg A log message describing a completed operation
+  # @param code The status code of the operation in the log message
+
+  def log(message, code = 0)
+    fields = {}.merge(@logfields)
+
+    fields['job']  = @job
+    fields['code'] = code
+
+    if not message.nil?
+      fields['message'] = message.sqlEscape()
+    end
+
+    keys   = []
+    values = []
+
+    fields.each do |k,v|
+      keys   << k.to_s
+      values << v
+    end
+
+    sql = %Q{insert into sys.log (#{keys.join(",")}) values ('#{values.join("','")}')}
+    @logdb.exec(sql)
+
+    $stderr.puts message
+  end
+
+end # module Util
+
+end # module NixAdm

data/src/lib/nixadm/version.rb

@@ -0,0 +1,11 @@
+# Version information. Auto-generated by CMake.
+module NixAdm
+
+VERSION_MAJ  = '1'
+VERSION_MIN  = '0'
+VERSION_CL   = ''
+VERSION_PL   = '6'
+VERSION      = '1.0.6-1'
+RELEASE_DATE = 'Thu, 05 Nov 2020 13:38:26 -0600'
+
+end # module NixAdm

data/src/lib/nixadm/zfs.rb

@@ -0,0 +1,564 @@
+require 'nixadm/pipeline'
+
+module NixAdm
+module ZFS
+
+class Host < NixAdm::Command
+
+  attr_reader :name
+
+  def initialize(name, port=22)
+    super(name, port)
+
+    @name = name
+  end
+
+  def exec(command)
+    run resolveCommand(command)
+  end
+
+  def pools()
+    exec 'zpool list -Hp'
+
+    data = @sys.out.split("\n")
+
+    names = []
+    data.each do |rec|
+      names << rec.split()[0]
+    end
+
+    return names
+  end
+
+  def pool(name)
+    return Pool.new(self, name)
+  end
+
+end
+
+class Object < Status
+
+  attr_reader :name, :host
+
+  def initialize(host, name)
+    super()
+
+    if host.is_a?(NixAdm::ZFS::Host) == false
+      raise 'host must be of class Host'
+    end
+
+    @host = host
+    @name = name
+  end
+
+  def to_s()
+    return @name
+  end
+
+  def <=>(other)
+    @name <=> other.name
+  end
+
+  def >(other)
+    @name > other.name
+  end
+
+  def <(other)
+    @name < other.name
+  end
+
+end
+
+class Pool < Object
+
+  def initialize(host, name)
+    super
+  end
+
+  def filesystems()
+    data = filesystemNames()
+
+    filter = []
+    data.each do |object|
+      filter << Filesystem.new(@host, self, object[:name])
+    end
+
+    return filter.sort
+  end
+
+  def volumes()
+    data = volumeNames()
+
+    filter = []
+    data.each do |object|
+      filter << Volume.new(@host, self, object[:name])
+    end
+
+    return filter.sort
+  end
+
+  def objects()
+    data = objectNames()
+
+    filter = []
+    data.each do |object|
+      type = object[:type]
+
+      if type == 'volume'
+        filter << Volume.new(@host, self, object[:name])
+      elsif type == 'filesystem'
+        filter << Filesystem.new(@host, self, object[:name])
+      else
+        filter << ZfsEntity.new(@host, self, object[:name])
+      end
+    end
+
+    return filter.sort
+  end
+
+  # Given filesystem name, return corresponding Filesystem object.
+  #
+  # @param name Filesystem name (excluding pool)
+  def filesystem(name)
+    data = filesystemNames()
+
+    if not data.include?(name)
+      return nil
+    end
+
+    return Filesystem.new(@host, self, name)
+  end
+
+  # Given volume name, return corresponding Volume object.
+  #
+  # @param name Volume name (excluding pool)
+  def volume(name)
+    data = volumeNames()
+
+    if not data.include?(name)
+      return nil
+    end
+
+    return Volume.new(@host, self, name)
+  end
+
+  # Given object name, return corresponding ZFS object.
+  #
+  # @param name Object name (excluding pool)
+  def object(name)
+    data = objectNames()
+
+    if not data.include?(name)
+      return nil
+    end
+
+    type = data[name]
+
+    if type == 'volume'
+      return Volume.new(@host, self, name)
+    elsif type == 'filesystem'
+      return Filesystem.new(@host, self, name)
+    end
+
+    return ZfsEntity.new(@host, self, name)
+  end
+
+  def objects(type='all')
+    @host.exec("zfs list -H -o name,type -t #{type} -r #{@name}")
+    data = @host.sys.out.split("\n")
+
+    names = []
+    data.each do |e|
+      n, t = e.split(' ')
+      names << { name: n, type: t }
+    end
+
+    return names
+  end
+
+  def createFilesystem(name)
+    return @host.exec("zfs create #{@name}/#{name}")
+  end
+
+  def createVolume(name)
+    return @host.exec("zfs create -V 32K #{@name}/#{name}")
+  end
+
+  private
+
+  def zfsEntities(type)
+    data = objects(type)
+
+    filter = {}
+    data.each do |entry|
+      e = entry[:name].split('/')[1..-1].join('/')
+      if e.size > 0
+        filter[e] = entry[:type]
+      end
+    end
+
+    return filter
+  end
+
+  def objectNames()
+    return zfsEntities('all')
+  end
+
+  def filesystemNames()
+    return zfsEntities('filesystem')
+  end
+
+  def volumeNames()
+    return zfsEntities('volume')
+  end
+
+end
+
+class ZfsEntity < Object
+
+  attr_reader :pool, :host
+
+  def initialize(host, pool, name)
+    super host, name
+
+    if pool.is_a?(Pool) == false
+      raise 'Second argument must be pool'
+    end
+
+    @pool = pool
+  end
+
+  def snapshot(id)
+    snap_names = snapshotNames()
+
+    snap_names.each do |name|
+      if name.split('@')[1].to_i == id
+        return snapshotInstance(name)
+      end
+    end
+
+    return nil
+  end
+
+  def snapshots()
+    snap_names = snapshotNames()
+
+    objects = []
+    snap_names.each do |name|
+      snapshot = snapshotInstance(name)
+
+      if block_given?
+        yield snapshot
+      else
+        objects << snapshot
+      end
+    end
+
+    return objects
+  end
+
+  # Convert snapshot integers to full names
+  def snapshotNames()
+    return filterSnapshots(fetchSnapshots())
+  end
+
+  def snapshotIds()
+    snaps = snapshotNames()
+
+    # We assume that relavant snapshot names are numerical values (epoch
+    # times). Any name containing alpha characters will be reduced to 0 by
+    # to_i().
+    ids = snaps.collect { |s| s.split('@')[1].to_i }
+
+    # Filter out the zero integer value entries, if any
+    ids.select! { |s| s > 0 }
+
+    return ids.sort
+  end
+
+  def lastSnapshotId()
+    return snapshotIds()[-1]
+  end
+
+  def lastSnapshot()
+    id = lastSnapshotId()
+
+    return nil if id.nil?
+
+    return snapshot(id)
+  end
+
+  def snapshotName(ss)
+    return "#{@pool.name}/#{@name}@#{ss}"
+  end
+
+  # Makes testing easier
+  def snapshotInstance(name)
+    return Snapshot.new(self, name)
+  end
+
+  def previousSnapshotId(id)
+    ids = snapshotIds()
+
+    prev = nil
+    ids.each do |x|
+      if x == id
+        return prev
+      end
+      prev = x
+    end
+  end
+
+  # Creates a snapshot using epoch time as name
+  def createSnapshot(id=nil)
+    name = newSnapshotName(id)
+
+    command = @host.resolveCommand("zfs snapshot #{name}")
+    if @host.run(command) == false
+      raise "Failed to create snapshot #{name}"
+    end
+
+    return name.split('@')[1].to_i
+  end
+
+  def deleteSnapshot(id)
+    name    = snapshotName(id)
+    command = @host.resolveCommand("zfs destroy #{name}")
+
+    if @host.run(command) == false
+      raise "Failed to create snapshot #{name}"
+    end
+  end
+
+  def deleteAllSnapshots()
+    snapshotIds().each do |id|
+      if deleteSnapshot(id) == false
+        return status()
+      end
+    end
+
+    return success()
+  end
+
+  # Delete all except the most recent snapshot
+  def trimSnapshots()
+    snapshotIds()[0..-2].each do |id|
+      deleteSnapshot(id)
+    end
+
+    return success()
+  end
+
+  # Given a set of snapshots in order, returns the most recent match if one
+  # exists.
+  def latestMatchingSnapshotId(ids)
+    my_ids = snapshotIds()
+    ids.reverse_each do |id|
+      if my_ids.include?(id)
+        return id
+      end
+    end
+
+    return nil
+  end
+
+  private
+
+  def fetchSnapshots()
+    return @pool.objects('snapshot')
+  end
+
+  def newSnapshotName(id=nil)
+    if not id.nil?
+      return "#{@pool.name}/#{@name}@#{id}"
+    end
+
+    return "#{@pool.name}/#{@name}@#{Time.now.to_i}"
+  end
+
+  def filterSnapshots(data)
+    filter = []
+    data.each do |entry|
+      fs = entry[:name].split('/')[1..-1].join('/')
+      if fs.match(/^#{@name}@/) != nil
+        filter << fs
+      end
+    end
+
+    return filter
+  end
+
+end
+
+class Filesystem < ZfsEntity
+
+  def initialize(host, pool, name)
+    super
+  end
+
+end
+
+class Volume < ZfsEntity
+
+  def initialize(host, pool, name)
+    super
+  end
+
+end
+
+class Snapshot < Object
+
+  attr_reader :object, :host
+
+  def initialize(object, name)
+
+    if object.is_a?(ZfsEntity) == false
+      raise 'First argument must be filesystem or zvol'
+    end
+
+    super object.host, name
+
+    @object = object
+  end
+
+  def id()
+    return @name.split('@')[1].to_i
+  end
+
+  def previous()
+    return @object.previousSnapshotId(id())
+  end
+
+  # Push this snapshot to other object
+  def send(object)
+
+    if object.is_a?(ZfsEntity) == false
+      raise 'First argument must be filesystem or zvol'
+    end
+
+    # Used for incremental if applicable
+    zfs_options = ''
+
+    # See if remote object has a matching previous snapshot to use as
+    # reference for incremental send
+    set = @object.snapshotIds()[0..-1]
+    previous_snap_id = object.latestMatchingSnapshotId(set)
+
+    if previous_snap_id == set[-1]
+      return true
+    end
+
+    if not previous_snap_id.nil?
+      # We can send an incremental snapshot
+      zfs_options = "-i #{previous_snap_id}"
+    else
+      # There are not previous reference snapshots we can use so we have to send
+      # a full snapshot. We may as well clear out all remote snapshots (if any)
+      # before doing do.
+      #
+      # Note: if this pukes for any reason, it should throw exception because
+      # there's no way we can handle a failure here.
+      object.deleteAllSnapshots()
+    end
+
+    # This sends either full or incremental depending on the value of
+    # zfs_options variable.
+
+    return push(object, zfs_options)
+  end
+
+  private
+
+  def push(object, zfs_options='')
+    snapshot    = "#{@object.pool.name}/#{@name}"
+    remote_port = object.pool.host.port
+    remote_host = object.pool.host.name
+
+    command_1 = "zfs send #{zfs_options} #{snapshot}"
+
+    command_2 = "ssh -p #{remote_port} #{remote_host} " +
+                "zfs receive -F #{object.pool.name}/#{object.name}"
+
+    command = @host.resolveCommand [ command_1, command_2 ]
+
+    return @host.run(command)
+  end
+
+end
+
+class Admin < Status
+
+  attr_reader :host
+
+  def initialize(host=nil, port=22)
+    super()
+
+    @host = Host.new(host, port)
+  end
+
+  # Given full pathname of ZFS object, return corresponding ZFS object. Returns
+  # nil upon no match.
+  def filesystem(name)
+    pool_name = name.split('/')[0]
+    pool = @host.pool(pool_name)
+
+    return nil if pool.nil?
+
+    fs_name = name.split('/')[1..-1].join('/')
+
+    return pool.filesystem(fs_name)
+  end
+
+  # Given full pathname of ZFS volumne, return corresponding Volume
+  # object. Returns nil upon no match.
+  def volume(name)
+    pool_name = name.split('/')[0]
+    pool = @host.pool(pool_name)
+
+    return nil if pool.nil?
+
+    vol_name = name.split('/')[1..-1].join('/')
+
+    return pool.volume(vol_name)
+  end
+
+  # Given full pathname of ZFS volumne, return corresponding Volume
+  # object. Returns nil upon no match.
+  def object(name)
+    pool_name = name.split('/')[0]
+    pool = @host.pool(pool_name)
+
+    return nil if pool.nil?
+
+    object_name = name.split('/')[1..-1].join('/')
+
+    return pool.object(object_name)
+  end
+
+  # Replicate changes on one ZFS filesystem on one host (src_fs) to another
+  # (dst_fs). Filesystem must be in sync. This means they must have most recent
+  # snapshots that match. If this is not met, function will abort.
+  #
+  # @param src_fs The source host. This is the machine that has the filesystem
+  # we want to replicate. Format can be in the form user@host or just
+  # host. Whatever is acceptable to ssh is acceptable here.
+  #
+  # @param dest_fs The destination host. This is the machine what will receive
+  # the delta from the source host, whose filesystem will be updated. Can be
+  # in the form user@host or just host. Whatever is acceptable to ssh is
+  # acceptable here.
+
+  def replicate(source_fs, dest_fs)
+    snapshot = source_fs.lastSnapshot()
+
+    if snapshot.nil?
+      return failure(-1, "No snapshots on filesystem #{sfs.name}")
+    end
+
+    return snapshot.send(dest_fs)
+  end
+
+end
+
+end # module ZFS
+end # module NixAdm