wakame-vdc-dcmgr 10.12.0 → 11.06.0

data/config/dcmgr.conf.example

@@ -3,12 +3,24 @@
 #------------------------
 
 # Database connection string
-database_url 'mysql://localhost/dcmgr?user=root'
+database_url 'mysql://localhost/wakame_dcmgr?user=root'
 # AMQP broker to be connected.
-amqp_server_uri 'amqp://192.168.2.25/'
+amqp_server_uri 'amqp://localhost/'
 
-# UUID for shared host pool.
-default_shared_host_pool 'tag-xxxxxxxx'
+# UUID for shared host pool or group.
+default_shared_host_pool 'tag-shhost'
 
-# UUID for shared network pool.
-default_shared_network_pool 'tag-xxxxxxxx'
+# UUID for shared network pool or group.
+default_shared_network_pool 'tag-shnet'
+
+# UUID for shared storage pool or group.
+default_shared_storage_pool 'tag-shstor'
+
+# system wide limit size in MB for creating new volume.
+# (not affect at cloning from snapshot)
+create_volume_max_size 3000
+create_volume_min_size 10
+
+# Default account quota limit
+account_instance_total_weight 100.0
+account_volume_total_size 100000 # 100GB

data/config/initializers/isono.rb

@@ -5,34 +5,18 @@ require 'eventmachine'
 
 Signal.trap('EXIT') { EventMachine.stop }
 
-if defined?(PhusionPassenger)
-  if PhusionPassenger::VERSION_STRING =~ /^3\.0\./
-   blk = proc { |forked|
-      if EventMachine.reactor_running?
-        EventMachine.stop
-        Dcmgr.class_eval {
-          @messaging_client = nil
-        }
-      end
-      Thread.new { EventMachine.epoll; EventMachine.run; }
-    }
-  else
-   blk = proc {
-      if EventMachine.reactor_running?
-        EventMachine.stop
-        Dcmgr.class_eval {
-          @messaging_client = nil
-        }
-      end
-      Thread.new { EventMachine.epoll; EventMachine.run; }
+def restart_reactor_and_messaging_client
+  if EventMachine.reactor_running?
+    EventMachine.stop
+    Dcmgr.class_eval {
+      @messaging_client = nil
     }
   end
-  PhusionPassenger.on_event(:starting_worker_process, &blk)
-else
-  EventMachine.stop if EventMachine.reactor_running?
   Thread.new { EventMachine.epoll; EventMachine.run; }
 end
 
+restart_reactor_and_messaging_client
+
 Dcmgr.class_eval {
   def self.messaging
     @messaging_client ||= Isono::MessagingClient.start(conf.amqp_server_uri) do

data/config/initializers/sequel.rb

@@ -1,15 +1,21 @@
 # -*- coding: utf-8 -*-
 
 require 'sequel'
-db = Sequel.connect(Dcmgr.conf.database_url)
+if Sequel::DATABASES.first.nil?
+  db = Sequel.connect(Dcmgr.conf.database_url)
+else
+  db = Sequel::DATABASES.first
+end
+
 #require 'logger' 
-#db.loggers << Logger.new(STDOUT)
+#db.loggers << Logger.new(STDERR)
 if db.is_a?(Sequel::MySQL::Database)
   Sequel::MySQL.default_charset = 'utf8'
   Sequel::MySQL.default_collate = 'utf8_general_ci'
   Sequel::MySQL.default_engine = 'InnoDB'
 
   db << "SET AUTOCOMMIT=0"
+  Dcmgr::Models::BaseNew.default_row_lock_mode = nil
 end
 
 # Disable TEXT to Sequel::SQL::Blob translation.
@@ -19,3 +25,6 @@ end
 [249, 250, 251, 252].each { |v|
   Sequel::MySQL::MYSQL_TYPES.delete(v)
 }
+
+# Set timezone to UTC
+Sequel.default_timezone = :utc

data/lib/dcmgr.rb

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
 
+require 'dcmgr/version'
+require 'ext/time'
+
 module Dcmgr
-  VERSION='10.12.0'
 
   class << self
     def conf
@@ -28,8 +30,16 @@ module Dcmgr
       self
     end
 
-    def run_initializers()
+    def run_initializers(*files)
       raise "Complete the configuration prior to run_initializers()." if @conf.nil?
+
+      @files ||= []
+      if files.length == 0
+        @files << "*"
+      else
+      	@files = files
+      end
+      
       initializer_hooks.each { |n|
         n.call
       }
@@ -42,29 +52,34 @@ module Dcmgr
       end
       @initializer_hooks
     end
-
   end
 
   initializer_hooks {
     Dcmgr.class_eval {
-      DCMGR_ROOT = ENV['DCMGR_ROOT'] || File.expand_path('../../', __FILE__)
+      unless defined?(DCMGR_ROOT)
+        DCMGR_ROOT = ENV['DCMGR_ROOT'] || File.expand_path('../../', __FILE__)
+      end
     }
   }
   
   # Add conf/initializers/*.rb loader 
   initializer_hooks {
     initializers_root = File.expand_path('config/initializers', DCMGR_ROOT) 
-    
-    if File.directory?(initializers_root)
-      Dir.glob("#{initializers_root}/*.rb") { |f|
-        ::Kernel.load(f)
-      }
-    end
+
+    @files.each { |file|  
+      if File.directory?(initializers_root)
+        Dir.glob("#{initializers_root}/#{file}.rb") { |f|
+          ::Kernel.load(f)
+        }
+      end
+    }
   }
   
   autoload :Logger, 'dcmgr/logger'
   
   module Models
+    class InvalidUUIDError < StandardError; end
+    class UUIDPrefixDuplication < StandardError; end
     autoload :Base, 'dcmgr/models/base'
 
     CREATE_TABLE_CLASSES=[:Account,:Tag,:TagMapping,:FrontendSystem,
@@ -74,7 +89,7 @@ module Dcmgr
                           :InstanceNetfilterGroup,
                           :InstanceSpec, :InstanceNic, :Network, :IpLease,
                           :SshKeyPair, :History, :HostnameLease, :MacLease,
-                          :VlanLease,
+                          :VlanLease, :Quota
                          ].freeze
     autoload :BaseNew, 'dcmgr/models/base_new'
     autoload :Account, 'dcmgr/models/account'
@@ -101,6 +116,7 @@ module Dcmgr
     autoload :HostnameLease, 'dcmgr/models/hostname_lease'
     autoload :MacLease, 'dcmgr/models/mac_lease'
     autoload :VlanLease, 'dcmgr/models/vlan_lease'
+    autoload :Quota, 'dcmgr/models/quota'
   end
 
   module Endpoints
@@ -112,6 +128,8 @@ module Dcmgr
     autoload :StaCollector, 'dcmgr/node_modules/sta_collector'
     autoload :HvaCollector, 'dcmgr/node_modules/hva_collector'
     autoload :InstanceHA, 'dcmgr/node_modules/instance_ha'
+    autoload :ServiceNetfilter, 'dcmgr/node_modules/service_netfilter'
+    autoload :InstanceMonitor, 'dcmgr/node_modules/instance_monitor'
   end
 
   module Stm
@@ -122,7 +140,48 @@ module Dcmgr
 
   module Helpers
     autoload :CliHelper, 'dcmgr/helpers/cli_helper'
+    autoload :NicHelper, 'dcmgr/helpers/nic_helper'
   end
 
   autoload :Tags, 'dcmgr/tags'
+
+  module Cli
+    require 'dcmgr/cli/errors'
+
+    autoload :Base, 'dcmgr/cli/base'
+    autoload :Network, 'dcmgr/cli/network'
+    autoload :Host, 'dcmgr/cli/host'
+    autoload :Storage, 'dcmgr/cli/storage'
+    autoload :AccountCli, 'dcmgr/cli/account'
+    autoload :UsersCli, 'dcmgr/cli/user'
+    autoload :Vlan, 'dcmgr/cli/vlan'
+    autoload :Image, 'dcmgr/cli/image'
+    autoload :KeyPair, 'dcmgr/cli/keypair'
+    autoload :Group, 'dcmgr/cli/group'
+    autoload :Spec, 'dcmgr/cli/spec'
+    autoload :Tag, 'dcmgr/cli/tag'
+    autoload :Quota, 'dcmgr/cli/quota'
+  end
+
+  module Rpc
+    autoload :HvaHandler, 'dcmgr/rpc/hva_handler'
+    autoload :KvmHelper, 'dcmgr/rpc/hva_handler'
+  end
+
+  # namespace for custom Rack HTTP middleware.
+  module Rack
+    autoload :RequestLogger, 'dcmgr/rack/request_logger'
+    autoload :RunInitializer, 'dcmgr/rack/run_initializer'
+  end
+  
+  module Drivers
+    autoload :SnapshotStorage, 'dcmgr/drivers/snapshot_storage'
+    autoload :S3Storage, 'dcmgr/drivers/s3_storage'
+    autoload :IIJGIOStorage, 'dcmgr/drivers/iijgio_storage'
+    autoload :Hypervisor, 'dcmgr/drivers/hypervisor'
+    autoload :Kvm , 'dcmgr/drivers/kvm'
+    autoload :Lxc , 'dcmgr/drivers/lxc'
+  end
+  
+  autoload :StorageService, 'dcmgr/storage_service'
 end

data/lib/dcmgr/cli/base.rb

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+
+require 'thor'
+
+module Dcmgr::Cli
+  class Base < Thor
+    protected
+    def self.basename
+      "#{super()} #{namespace}"
+    end
+
+    no_tasks {
+      public
+      # add before/after task hook.
+      def invoke_task(task, *args)
+        Dcmgr::Models::BaseNew.db.transaction do
+          before_task
+          super(task, *args)
+          after_task
+        end
+      end
+    
+      protected
+      def before_task
+      end
+      
+      def after_task
+      end
+    }
+    
+    def add(model,options)
+      raise ArgumentError unless options.is_a? Hash
+      #TODO: Make this check a little tighter by checking that the model is either from the wakame backend or frontend
+      #UnknownModelError.raise(model) unless model < Dcmgr::Models::BaseNew
+      UnknownModelError.raise(model) unless model < Sequel::Model
+      
+      fields = options.dup
+      
+      if fields.has_key?("uuid") || fields.has_key?(:uuid)
+        fields[:uuid] = model.trim_uuid(fields[:uuid]) if model.check_uuid_format(fields[:uuid])
+        Error.raise("UUID syntax invalid: #{fields[:uuid]}",100) unless model.check_trimmed_uuid_format(fields[:uuid])
+      end
+      
+      #Create database fields
+      new_record = model.create(fields)
+      
+      #Return uuid if there is one
+      new_record.canonical_uuid #if model.respond_to? "canonical_uuid"
+    end
+    
+    def del(model,uuid)
+      #UnknownModelError.raise(model) unless model < Dcmgr::Models::BaseNew
+      UnknownModelError.raise(model) unless model < Sequel::Model
+      to_delete = model[uuid] || UnknownUUIDError.raise(uuid)
+      to_delete.destroy
+    end
+    
+    def modify(model,uuid,fields)
+      #UnknownModelError.raise(model) unless model < Dcmgr::Models::BaseNew
+      UnknownModelError.raise(model) unless model < Sequel::Model
+      raise ArgumentError unless fields.is_a? Hash
+      to_modify = model[uuid] || UnknownUUIDError.raise(uuid)
+      
+      #Use a copy of the fields hash so this method can work with frozen hashes
+      fields_nonil = fields.merge({})
+      #Don't update empty fields
+      fields_nonil.delete_if {|key,value| value.nil?}
+      
+      to_modify.set(fields_nonil)
+      to_modify.updated_at = Time.now.utc.iso8601 if to_modify.with_timestamps?
+      to_modify.save_changes
+    end
+  end
+end

data/lib/dcmgr/cli/errors.rb

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr::Cli
+  class Error < StandardError
+    attr_reader :exit_code
+    def initialize(msg, exit_code=1)
+      super(msg)
+      @exit_code = exit_code
+    end
+
+    def self.raise(msg, exit_code)
+      Kernel.raise(if msg.is_a?(self)
+                     msg
+                   else
+                     self.new(msg, exit_code)
+                   end)
+    end
+  end
+  
+  class UnknownUUIDError < Error
+    def initialize(uuid,exit_code=100)
+      super("Unknown UUID: '#{uuid}'.")
+    end
+    
+    def self.raise(uuid,exit_code=100)
+      super
+    end
+  end
+  
+  class UnsupportedArchError < Error
+    def initialize(arch,exit_code=100)
+      super("Unsupported arch type: '#{arch}'.")
+    end
+    
+    def self.raise(arch,exit_code=100)
+      super
+    end
+  end
+  
+  class UnknownModelError < Error
+    def initialize(model,exit_code=100)
+      super("Not a sequel model: '#{model}'.")
+    end
+    
+    def self.raise(model,exit_code=100)
+      super
+    end
+  end
+
+  class UnsupportedHypervisorError < Error
+    def initialize(arch,exit_code=100)
+      super("Unsupported hypervisor type: '#{arch}'.")
+    end
+    
+    def self.raise(arch,exit_code=100)
+      super
+    end
+  end
+end

data/lib/dcmgr/cli/group.rb

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr::Cli
+  class Group < Base
+    namespace :group
+    M = Dcmgr::Models
+
+    desc "add [options]", "Add a new security group"
+    method_option :uuid, :type => :string, :aliases => "-u", :desc => "The UUID for the new security group."
+    method_option :account_id, :type => :string, :aliases => "-a", :desc => "The UUID of the account this security group belongs to.", :required => true
+    method_option :name, :type => :string, :aliases => "-n", :desc => "The name for this security group.", :required => true
+    method_option :description, :type => :string, :aliases => "-d", :desc => "The description for this new security group."
+    def add
+      UnknownUUIDError.raise(options[:account_id]) if M::Account[options[:account_id]].nil?
+      
+      puts super(M::NetfilterGroup,options)
+    end
+    
+    desc "del UUID", "Delete a security group"
+    def del(uuid)
+      super(M::NetfilterGroup,uuid)
+    end
+    
+    desc "show [UUID]", "Show security group(s)"
+    def show(uuid=nil)
+      if uuid
+        group = M::NetfilterGroup[uuid] || UnknownUUIDError.raise(uuid)
+        puts ERB.new(<<__END, nil, '-').result(binding)
+Group UUID:
+  <%= group.canonical_uuid %>
+Account id:
+  <%= group.account_id %>
+<%- if group.description -%>
+Description:
+  <%= group.description %>
+<%- end -%>
+<%- unless group.netfilter_rules.empty? -%>
+Rules:
+<%- group.netfilter_rules.each { |rule| -%>
+  <%= rule.permission %>
+<%- } -%>
+<%- end -%>
+__END
+      else
+        puts ERB.new(<<__END, nil, '-').result(binding)
+<%- M::NetfilterGroup.all { |row| -%>
+<%= row.canonical_uuid %>\t<%= row.account_id %>\t<%= row.name %>
+<%- } -%>
+__END
+      end
+    end
+    
+    desc "modify UUID [options]", "Modify an existing security group"
+    method_option :account_id, :type => :string, :aliases => "-a", :desc => "The UUID of the account this security group belongs to."
+    method_option :name, :type => :string, :aliases => "-n", :desc => "The name for this security group."
+    method_option :description, :type => :string, :aliases => "-d", :desc => "The description for this new security group."
+    def modify(uuid)
+      UnknownUUIDError.raise(options[:account_id]) if options[:account_id] && M::Account[options[:account_id]].nil?
+      super(M::NetfilterGroup,uuid,options)
+    end
+    
+    desc "apply UUID [options]", "Apply a security group to an instance"
+    method_option :instance, :type => :string, :aliases => "-i", :required => :true, :desc => "The instance to apply the group to"
+    def apply(uuid)
+      group = M::NetfilterGroup[uuid] || UnknownUUIDError.raise(uuid)
+      instance = M::Instance[options[:instance]] || UnknownUUIDError.raise(options[:instance])
+      Error.raise("Group #{uuid} is already applied to instance #{options[:instance]}.",100) if group.instances.member?(instance)
+      group.add_instance(instance)
+    end
+    
+    desc "remove UUID [options]", "Remove a security group from an instance"
+    method_option :instance, :type => :string, :aliases => "-i", :required => :true, :desc => "The instance to remove the group from"
+    def remove(uuid)
+      group = M::NetfilterGroup[uuid] || UnknownUUIDError.raise(uuid)
+      instance = M::Instance[options[:instance]] || UnknownUUIDError.raise(options[:instance])
+      Error.raise("Group #{uuid} is not applied to instance #{options[:instance]}.",100) unless group.instances.member?(instance)
+      group.remove_instance(instance)
+    end
+    
+    desc "addrule UUID [options]", "Add a rule to a security group"
+    method_option :rule, :type => :string, :aliases => "-r", :desc => "The new rule to be added"
+    def addrule(g_uuid)
+      UnknownUUIDError.raise(g_uuid) if M::NetfilterGroup[g_uuid].nil?
+      
+      #TODO: check rule syntax
+      new_rule = M::NetfilterRule.new(:permission => options[:rule])
+      new_rule.netfilter_group = M::NetfilterGroup[g_uuid]
+      new_rule.save
+    end
+    
+    desc "delrule UUID [options]", "Delete a rule from a security group"
+    method_option :rule, :type => :string, :aliases => "-r", :desc => "The rule to be deleted."
+    def delrule(g_uuid)
+      UnknownUUIDError.raise(g_uuid) if M::NetfilterGroup[g_uuid].nil?
+      rule = M::NetfilterRule.find(:netfilter_group_id => M::NetfilterGroup[g_uuid].id,:permission => options[:rule])
+      Error.raise("Group '#{g_uuid}' does not contain rule '#{options[:rule]}'.",100) if rule.nil?
+      
+      rule.destroy
+    end
+  end
+end