wakame-vdc-agents 10.12.0 → 11.06.0

data/bin/nsa

@@ -5,14 +5,11 @@
 # This agent aims to configure DNS/DHCP daemons or devices to supply
 # IP address and Hostname for Instances.
 
-begin
-  require 'rubygems'
-  require 'bundler'
-  Bundler.setup(:default)
-rescue Exception
-end
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 
-require File.expand_path('../../config/path_resolver', __FILE__)
+require 'dcmgr/rubygems'
+require 'dcmgr'
+require 'isono'
 
 require 'erb'
 require 'eventmachine'
@@ -168,8 +165,7 @@ manifest.instance_eval do
     c.network_name = nil
   end
 
-  config_path File.expand_path('config/nsa.conf', app_root)
-  load_config
+  load_config File.expand_path('config/nsa.conf', app_root)
 end
 
 

data/bin/sta

@@ -1,23 +1,21 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
 
-begin
-  require 'rubygems'
-  require 'bundler'
-  Bundler.setup(:default)
-rescue Exception
-end
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 
-require File.expand_path('../../config/path_resolver', __FILE__)
+require 'dcmgr/rubygems'
+require 'dcmgr'
+require 'isono'
 
 include Isono::Runner::RpcServer
 
 class ZfsHandler < EndpointBuilder
   include Dcmgr::Logger
-
+  include Dcmgr::Helpers::CliHelper
+  
   job :create_volume do
     volume_id = request.args[0]
-    job = Dcmgr::Stm::VolumeContext.new(volume_id)
+    dest = Dcmgr::StorageService.repository(request.args[1])
     data = rpc.request('sta-collector', 'get_volume', volume_id)
     raise "Invalid volume state: #{data[:state]}" unless data[:state].to_s == 'registering'
 
@@ -26,140 +24,195 @@ class ZfsHandler < EndpointBuilder
       raise "Invalid snapshot state: #{sdata[:state]}" unless sdata[:state].to_s == 'available'
     end
     logger.info("creating new volume #{volume_id}")
-    job.stm.state=data[:state].to_sym
-    job.stm.on_create
 
-    vol_path = "#{data[:storage_pool][:export_path]}/#{data[:export_path]}"
-    `zfs list #{File.dirname(vol_path)} > /dev/null 2>&1`
+    rpc.request('sta-collector', 'update_volume', volume_id, {:state=>:creating, :export_path=>data[:uuid]})
+
+    vol_path = "#{data[:storage_pool][:export_path]}/#{data[:uuid]}"
+    sh("/usr/sbin/zfs list %s > /dev/null 2>&1", [File.dirname(vol_path)])
     if $?.exitstatus != 0
       # create parent filesystem
-      `zfs create -p #{File.dirname(vol_path)}`
+      sh("/usr/sbin/zfs create -p %s", [File.dirname(vol_path)])
       logger.info("create parent filesystem: #{File.dirname(vol_path)}")
     end
 
     if sdata
+      zsnap_dir = "#{data[:storage_pool][:snapshot_base_path]}/#{sdata[:account_id]}/" 
+      zsnap_filename = "#{sdata[:uuid]}.zsnap"
+      zsnap_file = File.join(zsnap_dir, zsnap_filename)
+      
+      unless File.exists?(zsnap_file)
+        if Dcmgr::StorageService.has_driver?(dest[:driver])
+          begin
+            storage = Dcmgr::StorageService.new(dest[:driver], dest[:access_key], dest[:secret_key])
+            snapshot_file = "#{dest[:path]}#{dest[:filename]}"
+            bucket = storage.bucket(dest[:bucket])
+            bucket.download(snapshot_file, zsnap_filename, zsnap_dir)
+            logger.info("download to #{dest[:driver]}: #{snapshot_file}")
+         rescue => e 
+            logger.error(e.message)
+            raise "snapshot not downloaded" if $?.exitstatus != 0
+          end
+        end
+      end
+      
       # create volume from snapshot
-      v = `zfs receive #{vol_path} < #{data[:storage_pool][:snapshot_base_path]}/#{sdata[:account_id]}/#{sdata[:uuid]}.zsnap`
-      if $?.exitstatus != 0
-        raise "volume already exists: #{volume_id}"
+      if File.exists?(zsnap_file)
+        sh("/usr/sbin/zfs receive %s < %s", [vol_path, zsnap_file])
+        if $?.exitstatus != 0
+          raise "volume already exists: #{volume_id}"
+        end
+      else
+        raise "snapshot file isn't exists: #{zsnap_file}"
       end
 
-      v = `zfs destroy #{vol_path}@#{sdata[:uuid]}`
+      sh("/usr/sbin/zfs destroy %s@%s", [vol_path, sdata[:uuid]])
       if $?.exitstatus != 0
         raise "volume snapshot has not deleted: #{volume_id}@#{sdata[:uuid]}"
       end
 
-      vl = `zfs list #{vol_path}`
-      if vl.nil? || $?.exitstatus != 0
+      sh("/usr/sbin/zfs list %s", [vol_path])
+      if $?.exitstatus != 0
         raise "volume has not be created: #{volume_id}"
       end
+      
+      sh("rm -rf %s", ["#{data[:storage_pool][:snapshot_base_path]}/#{sdata[:account_id]}/#{sdata[:uuid]}.zsnap"])
+      logger.info("delete file: " + File.join(zsnap_dir, zsnap_file))
+       
+      if $?.exitstatus != 0
+        raise "snapshot file cna't deleted"
+      end
     else
       # create volume
-      v = `zfs create -p -V #{data[:size]}m #{vol_path}`
+      sh("/usr/sbin/zfs create -p -V %s %s", ["#{data[:size]}m", vol_path])
       if $?.exitstatus != 0
         raise "volume already exists: #{volume_id}"
       end
-      vl = `zfs list #{vol_path}`
-      if vl.nil? || $?.exitstatus != 0
+      sh("/usr/sbin/zfs list %s", [vol_path])
+      if $?.exitstatus != 0
         raise "volume has not be created: #{volume_id}"
       end
     end
 
-    rpc.request('sta-collector', 'update_volume', job.to_hash(:export_path=>data[:export_path]))
     logger.info("created new volume: #{volume_id}")
 
-    job.stm.on_register
-    vr = `zfs shareiscsi=on #{data[:storage_pool][:export_path]}/#{data[:uuid]}`
+    sh("/usr/sbin/zfs shareiscsi=on %s/%s", [data[:storage_pool][:export_path], data[:uuid]])
     if $?.exitstatus != 0
       raise "failed iscsi target request: #{volume_id}"
     end
-    il = `iscsitadm list target -v #{data[:storage_pool][:export_path]}/#{data[:uuid]}`
+    il = sh("iscsitadm list target -v %s", ["#{data[:storage_pool][:export_path]}/#{data[:uuid]}"])
     if $?.exitstatus != 0
       raise "iscsi target has not be created #{volume_id}"
     end
-    il = il.downcase.split("\n").select {|row| row.strip!}
+    il = il[:stdout].downcase.split("\n").select {|row| row.strip!}
     # :transport_information => {:iqn => "iqn.1986-03.com.sun:02:787bca42-9639-44e4-f115-f5b06ed31817", :lun => 0}
     opt = {:iqn => il[0].split(": ").last, :lun=>il[6].split(": ").last.to_i}
 
-    rpc.request('sta-collector', 'update_volume', job.to_hash(:transport_information=>opt))
+    rpc.request('sta-collector', 'update_volume', volume_id, {:state=>:available, :transport_information=>opt})
     logger.info("registered iscsi target: #{volume_id}")
   end
 
   job :delete_volume do
     volume_id = request.args[0]
-    job = Dcmgr::Stm::VolumeContext.new(volume_id)
     data = rpc.request('sta-collector', 'get_volume', volume_id)
-    logger.info("deleting volume: #{volume_id}")
-    raise "Invalid volume state: #{data[:state]}" unless data[:state].to_s == 'deregistering'
-    job.stm.state=data[:state].to_sym
-
+    logger.info("#{volume_id}: start deleting volume.")
+    errcount = 0
+    if data[:state].to_s == 'deleted'
+      raise "#{volume_id}: Invalid volume state: deleted"
+    end
+    if data[:state].to_s != 'deregistering'
+      logger.warn("#{volume_id}: Unexpected volume state but try again: #{data[:state]}")
+    end
+    
     # deregisterd iscsi target
-    job.stm.on_delete
-    vr = `zfs shareiscsi=off #{data[:storage_pool][:export_path]}/#{data[:export_path]}`
-    il = `iscsitadm list target #{data[:storage_pool][:export_path]}/#{data[:export_path]}`
-    unless il.empty?
-      raise "iscsi target has not deleted: #{volume_id} iqn: #{data[:transport_information][:iqn]}"
+    begin
+      sh("/usr/sbin/zfs shareiscsi=off %s", ["#{data[:storage_pool][:export_path]}/#{data[:uuid]}"])
+    rescue => e
+      logger.error("#{volume_id}: Failed to delete ISCSI target entry.")
+      errcount += 1
     end
-
-    rpc.request('sta-collector', 'update_volume', job.to_hash)
-    logger.info("deregistered iscsi target: #{volume_id} iqn: #{data[:transport_information][:iqn]}")
-
+    
+    rpc.request('sta-collector', 'update_volume', volume_id, {:state=>:deleting})
+    
     # delete volume
-    job.stm.on_delete
-    job.on_delete
-    v = `zfs destroy #{data[:storage_pool][:export_path]}/#{data[:export_path]}`
-    vl = `zfs list #{data[:storage_pool][:export_path]}/#{data[:export_path]}`
-    unless vl.empty?  
-      raise "volume has not deleted: #{volume_id}"
+    begin 
+      sh("/usr/sbin/zfs destroy %s", ["#{data[:storage_pool][:export_path]}/#{data[:uuid]}"])
+    rescue => e
+      logger.error("#{volume_id}: Failed to delete zfs volume: #{data[:storage_pool][:export_path]}/#{data[:uuid]}")
+      errcount += 1
     end
 
-    rpc.request('sta-collector', 'update_volume', job.to_hash)
-    logger.info("deleted volume: #{volume_id}")
+    rpc.request('sta-collector', 'update_volume', volume_id, {:state=>:deleted, :deleted_at=>Time.now.utc})
+    if errcount > 0
+      logger.info("#{volume_id}: Encountered one or more errors during deleting.")
+    else
+      logger.info("#{volume_id}: Deleted volume successfully.")
+    end
   end
 
   job :create_snapshot do
     snapshot_id = request.args[0]
-    job = Dcmgr::Stm::SnapshotContext.new(snapshot_id)
+    dest = Dcmgr::StorageService.repository(request.args[1])
     sdata = rpc.request('sta-collector', 'get_snapshot', snapshot_id) unless snapshot_id.nil?
     data = rpc.request('sta-collector', 'get_volume', sdata[:origin_volume_id])
     logger.info("create new snapshot: #{snapshot_id}")
-    raise "Invalid volume state: #{data[:state]}" unless data[:state].to_s == 'available'
-    job.stm.state=sdata[:state].to_sym
-    job.stm.on_create
+    raise "Invalid volume state: #{data[:state]}" unless data[:state].to_s == 'available' || data[:state].to_s == 'attached'
 
-    vol_path = "#{data[:storage_pool][:export_path]}/#{data[:export_path]}"
+    vol_path = "#{data[:storage_pool][:export_path]}/#{data[:uuid]}"
     snap_dir = "#{data[:storage_pool][:snapshot_base_path]}/#{sdata[:account_id]}"
     unless File.exist?(snap_dir)
       # create a directory to save snapshot
       `mkdir -p #{snap_dir}`
       logger.info("create a directory: #{snap_dir}")
     end
-    snap = `zfs snapshot #{vol_path}@#{sdata[:uuid]}`
-    rpc.request('sta-collector', 'update_snapshot', job.to_hash)
-    logger.info("creating new snapshot: #{snapshot_id}")
-
-    job.stm.on_create
-    snap_send = `zfs send #{vol_path}@#{sdata[:uuid]} > #{snap_dir}/#{sdata[:uuid]}.zsnap`
-    snap_delete = `zfs destroy #{vol_path}@#{sdata[:uuid]}`
+    sh("/usr/sbin/zfs snapshot %s@%s", [vol_path, sdata[:uuid]])
+    zsnap_file = "#{snap_dir}/#{sdata[:uuid]}.zsnap"
+    sh("/usr/sbin/zfs send %s@%s > %s", [vol_path, sdata[:uuid], zsnap_file])
+    sh("/usr/sbin/zfs destroy %s@%s", [vol_path, sdata[:uuid]])
+    
+    if Dcmgr::StorageService.has_driver?(dest[:driver])
+      begin
+        storage = Dcmgr::StorageService.new(dest[:driver], dest[:access_key], dest[:secret_key])
+        snapshot_file = "#{dest[:path]}#{dest[:filename]}"
+        bucket = storage.bucket(dest[:bucket])
+        bucket.upload(snapshot_file, "#{zsnap_file}")    
+        logger.info("upload to #{dest[:driver]}: #{snapshot_file}")
+      rescue => e 
+        logger.error(e.message)
+        raise "snapshot has not be uploaded" if $?.exitstatus != 0
+      ensure
+        sh("rm -rf %s", ["#{data[:storage_pool][:snapshot_base_path]}/#{sdata[:account_id]}/#{sdata[:uuid]}.zsnap"])
+      end
+    end
 
-    rpc.request('sta-collector', 'update_snapshot', job.to_hash)
+    rpc.request('sta-collector', 'update_snapshot', snapshot_id, {:state=>:available})
     logger.info("created new snapshot: #{snapshot_id}")
   end
 
   job :delete_snapshot do
     snapshot_id = request.args[0]
-    job = Dcmgr::Stm::SnapshotContext.new(snapshot_id)
+    dest = Dcmgr::StorageService.repository(request.args[1])
     sdata = rpc.request('sta-collector', 'get_snapshot', snapshot_id)
     data = rpc.request('sta-collector', 'get_volume', sdata[:origin_volume_id])
     logger.info("deleting snapshot: #{snapshot_id}")
     raise "Invalid snapshot state: #{sdata[:state]}" unless sdata[:state].to_s == 'deleting'
-    job.stm.state=sdata[:state].to_sym
-    job.stm.on_delete
-
-    system("rm -rf #{data[:storage_pool][:snapshot_base_path]}/#{sdata[:account_id]}/#{sdata[:uuid]}.zsnap")
-    raise "snapshot has not be deleted" if $?.exitstatus != 0
+    
+    if Dcmgr::StorageService.has_driver?(dest[:driver])
+      begin
+        storage = Dcmgr::StorageService.new(dest[:driver], dest[:access_key], dest[:secret_key])
+        snapshot_file = "#{dest[:path]}#{dest[:filename]}"
+        bucket = storage.bucket(dest[:bucket])
+        bucket.delete(snapshot_file)    
+        logger.info("delete sanpshot file from #{dest[:driver]}: #{snapshot_file}")
+      rescue => e 
+        logger.error(e.message)
+        raise "snapshot has not be deleted" if $?.exitstatus != 0
+      end
+    else
+      sh("rm -rf %s", ["#{data[:storage_pool][:snapshot_base_path]}/#{sdata[:account_id]}/#{sdata[:uuid]}.zsnap"])
+      raise "snapshot has not be deleted" if $?.exitstatus != 0
+    end
 
-    rpc.request('sta-collector', 'update_snapshot', job.to_hash)
+    rpc.request('sta-collector', 'update_snapshot', snapshot_id, {:state=>:deleted, :deleted_at=>Time.now.utc})
     logger.info("deleted snapshot: #{snapshot_id}")
   end
 

data/config/hva.conf.example

@@ -17,3 +17,15 @@ config.bridge_prefix   = 'br'
 
 # bridge device name novlan
 config.bridge_novlan   = 'br0'
+
+# display netfitler commands
+config.verbose_netfilter = false
+
+# netfilter log output flag
+config.packet_drop_log = false
+
+# debug netfilter
+config.debug_iptables = false
+
+# Use ipset for netfilter
+config.use_ipset       = false

data/config/initializers/isono.rb

@@ -5,34 +5,18 @@ require 'eventmachine'
 
 Signal.trap('EXIT') { EventMachine.stop }
 
-if defined?(PhusionPassenger)
-  if PhusionPassenger::VERSION_STRING =~ /^3\.0\./
-   blk = proc { |forked|
-      if EventMachine.reactor_running?
-        EventMachine.stop
-        Dcmgr.class_eval {
-          @messaging_client = nil
-        }
-      end
-      Thread.new { EventMachine.epoll; EventMachine.run; }
-    }
-  else
-   blk = proc {
-      if EventMachine.reactor_running?
-        EventMachine.stop
-        Dcmgr.class_eval {
-          @messaging_client = nil
-        }
-      end
-      Thread.new { EventMachine.epoll; EventMachine.run; }
+def restart_reactor_and_messaging_client
+  if EventMachine.reactor_running?
+    EventMachine.stop
+    Dcmgr.class_eval {
+      @messaging_client = nil
     }
   end
-  PhusionPassenger.on_event(:starting_worker_process, &blk)
-else
-  EventMachine.stop if EventMachine.reactor_running?
   Thread.new { EventMachine.epoll; EventMachine.run; }
 end
 
+restart_reactor_and_messaging_client
+
 Dcmgr.class_eval {
   def self.messaging
     @messaging_client ||= Isono::MessagingClient.start(conf.amqp_server_uri) do

data/config/initializers/sequel.rb

@@ -1,15 +1,21 @@
 # -*- coding: utf-8 -*-
 
 require 'sequel'
-db = Sequel.connect(Dcmgr.conf.database_url)
+if Sequel::DATABASES.first.nil?
+  db = Sequel.connect(Dcmgr.conf.database_url)
+else
+  db = Sequel::DATABASES.first
+end
+
 #require 'logger' 
-#db.loggers << Logger.new(STDOUT)
+#db.loggers << Logger.new(STDERR)
 if db.is_a?(Sequel::MySQL::Database)
   Sequel::MySQL.default_charset = 'utf8'
   Sequel::MySQL.default_collate = 'utf8_general_ci'
   Sequel::MySQL.default_engine = 'InnoDB'
 
   db << "SET AUTOCOMMIT=0"
+  Dcmgr::Models::BaseNew.default_row_lock_mode = nil
 end
 
 # Disable TEXT to Sequel::SQL::Blob translation.
@@ -19,3 +25,6 @@ end
 [249, 250, 251, 252].each { |v|
   Sequel::MySQL::MYSQL_TYPES.delete(v)
 }
+
+# Set timezone to UTC
+Sequel.default_timezone = :utc

data/lib/dcmgr.rb

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
 
+require 'dcmgr/version'
+require 'ext/time'
+
 module Dcmgr
-  VERSION='10.12.0'
 
   class << self
     def conf
@@ -28,8 +30,16 @@ module Dcmgr
       self
     end
 
-    def run_initializers()
+    def run_initializers(*files)
       raise "Complete the configuration prior to run_initializers()." if @conf.nil?
+
+      @files ||= []
+      if files.length == 0
+        @files << "*"
+      else
+      	@files = files
+      end
+      
       initializer_hooks.each { |n|
         n.call
       }
@@ -42,29 +52,34 @@ module Dcmgr
       end
       @initializer_hooks
     end
-
   end
 
   initializer_hooks {
     Dcmgr.class_eval {
-      DCMGR_ROOT = ENV['DCMGR_ROOT'] || File.expand_path('../../', __FILE__)
+      unless defined?(DCMGR_ROOT)
+        DCMGR_ROOT = ENV['DCMGR_ROOT'] || File.expand_path('../../', __FILE__)
+      end
     }
   }
   
   # Add conf/initializers/*.rb loader 
   initializer_hooks {
     initializers_root = File.expand_path('config/initializers', DCMGR_ROOT) 
-    
-    if File.directory?(initializers_root)
-      Dir.glob("#{initializers_root}/*.rb") { |f|
-        ::Kernel.load(f)
-      }
-    end
+
+    @files.each { |file|  
+      if File.directory?(initializers_root)
+        Dir.glob("#{initializers_root}/#{file}.rb") { |f|
+          ::Kernel.load(f)
+        }
+      end
+    }
   }
   
   autoload :Logger, 'dcmgr/logger'
   
   module Models
+    class InvalidUUIDError < StandardError; end
+    class UUIDPrefixDuplication < StandardError; end
     autoload :Base, 'dcmgr/models/base'
 
     CREATE_TABLE_CLASSES=[:Account,:Tag,:TagMapping,:FrontendSystem,
@@ -74,7 +89,7 @@ module Dcmgr
                           :InstanceNetfilterGroup,
                           :InstanceSpec, :InstanceNic, :Network, :IpLease,
                           :SshKeyPair, :History, :HostnameLease, :MacLease,
-                          :VlanLease,
+                          :VlanLease, :Quota
                          ].freeze
     autoload :BaseNew, 'dcmgr/models/base_new'
     autoload :Account, 'dcmgr/models/account'
@@ -101,6 +116,7 @@ module Dcmgr
     autoload :HostnameLease, 'dcmgr/models/hostname_lease'
     autoload :MacLease, 'dcmgr/models/mac_lease'
     autoload :VlanLease, 'dcmgr/models/vlan_lease'
+    autoload :Quota, 'dcmgr/models/quota'
   end
 
   module Endpoints
@@ -112,6 +128,8 @@ module Dcmgr
     autoload :StaCollector, 'dcmgr/node_modules/sta_collector'
     autoload :HvaCollector, 'dcmgr/node_modules/hva_collector'
     autoload :InstanceHA, 'dcmgr/node_modules/instance_ha'
+    autoload :ServiceNetfilter, 'dcmgr/node_modules/service_netfilter'
+    autoload :InstanceMonitor, 'dcmgr/node_modules/instance_monitor'
   end
 
   module Stm
@@ -122,7 +140,48 @@ module Dcmgr
 
   module Helpers
     autoload :CliHelper, 'dcmgr/helpers/cli_helper'
+    autoload :NicHelper, 'dcmgr/helpers/nic_helper'
   end
 
   autoload :Tags, 'dcmgr/tags'
+
+  module Cli
+    require 'dcmgr/cli/errors'
+
+    autoload :Base, 'dcmgr/cli/base'
+    autoload :Network, 'dcmgr/cli/network'
+    autoload :Host, 'dcmgr/cli/host'
+    autoload :Storage, 'dcmgr/cli/storage'
+    autoload :AccountCli, 'dcmgr/cli/account'
+    autoload :UsersCli, 'dcmgr/cli/user'
+    autoload :Vlan, 'dcmgr/cli/vlan'
+    autoload :Image, 'dcmgr/cli/image'
+    autoload :KeyPair, 'dcmgr/cli/keypair'
+    autoload :Group, 'dcmgr/cli/group'
+    autoload :Spec, 'dcmgr/cli/spec'
+    autoload :Tag, 'dcmgr/cli/tag'
+    autoload :Quota, 'dcmgr/cli/quota'
+  end
+
+  module Rpc
+    autoload :HvaHandler, 'dcmgr/rpc/hva_handler'
+    autoload :KvmHelper, 'dcmgr/rpc/hva_handler'
+  end
+
+  # namespace for custom Rack HTTP middleware.
+  module Rack
+    autoload :RequestLogger, 'dcmgr/rack/request_logger'
+    autoload :RunInitializer, 'dcmgr/rack/run_initializer'
+  end
+  
+  module Drivers
+    autoload :SnapshotStorage, 'dcmgr/drivers/snapshot_storage'
+    autoload :S3Storage, 'dcmgr/drivers/s3_storage'
+    autoload :IIJGIOStorage, 'dcmgr/drivers/iijgio_storage'
+    autoload :Hypervisor, 'dcmgr/drivers/hypervisor'
+    autoload :Kvm , 'dcmgr/drivers/kvm'
+    autoload :Lxc , 'dcmgr/drivers/lxc'
+  end
+  
+  autoload :StorageService, 'dcmgr/storage_service'
 end