wakame-vdc-dcmgr 11.06.0 → 11.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (136) hide show
  1. data/Rakefile +19 -31
  2. data/bin/collector +6 -1
  3. data/config/db/migrations/0001_v1110_origin.rb +446 -0
  4. data/config/dcmgr.conf.example +51 -0
  5. data/lib/dcmgr.rb +99 -22
  6. data/lib/dcmgr/cli/base.rb +34 -1
  7. data/lib/dcmgr/cli/host.rb +24 -20
  8. data/lib/dcmgr/cli/image.rb +38 -19
  9. data/lib/dcmgr/cli/keypair.rb +16 -12
  10. data/lib/dcmgr/cli/network.rb +189 -81
  11. data/lib/dcmgr/cli/quota.rb +2 -2
  12. data/lib/dcmgr/cli/security_group.rb +106 -0
  13. data/lib/dcmgr/cli/spec.rb +144 -39
  14. data/lib/dcmgr/cli/storage.rb +16 -15
  15. data/lib/dcmgr/cli/tag.rb +20 -14
  16. data/lib/dcmgr/cli/vlan.rb +5 -5
  17. data/lib/dcmgr/drivers/backing_store.rb +32 -0
  18. data/lib/dcmgr/drivers/comstar.rb +81 -0
  19. data/lib/dcmgr/drivers/iijgio_storage.rb +9 -19
  20. data/lib/dcmgr/drivers/iscsi_target.rb +41 -0
  21. data/lib/dcmgr/drivers/kvm.rb +161 -28
  22. data/lib/dcmgr/drivers/linux_iscsi.rb +60 -0
  23. data/lib/dcmgr/drivers/local_storage.rb +24 -0
  24. data/lib/dcmgr/drivers/lxc.rb +167 -125
  25. data/lib/dcmgr/drivers/raw.rb +74 -0
  26. data/lib/dcmgr/drivers/s3_storage.rb +7 -19
  27. data/lib/dcmgr/drivers/snapshot_storage.rb +18 -28
  28. data/lib/dcmgr/drivers/storage_initiator.rb +28 -0
  29. data/lib/dcmgr/drivers/sun_iscsi.rb +32 -0
  30. data/lib/dcmgr/drivers/zfs.rb +77 -0
  31. data/lib/dcmgr/endpoints/core_api.rb +315 -263
  32. data/lib/dcmgr/endpoints/errors.rb +21 -10
  33. data/lib/dcmgr/endpoints/metadata.rb +360 -23
  34. data/lib/dcmgr/helpers/cli_helper.rb +6 -3
  35. data/lib/dcmgr/helpers/ec2_metadata_helper.rb +9 -0
  36. data/lib/dcmgr/helpers/nic_helper.rb +11 -0
  37. data/lib/dcmgr/helpers/snapshot_storage_helper.rb +34 -0
  38. data/lib/dcmgr/models/account.rb +0 -6
  39. data/lib/dcmgr/models/account_resource.rb +0 -4
  40. data/lib/dcmgr/models/base_new.rb +14 -2
  41. data/lib/dcmgr/models/dhcp_range.rb +38 -0
  42. data/lib/dcmgr/models/frontend_system.rb +0 -6
  43. data/lib/dcmgr/models/history.rb +0 -11
  44. data/lib/dcmgr/models/host_node.rb +131 -0
  45. data/lib/dcmgr/models/hostname_lease.rb +0 -8
  46. data/lib/dcmgr/models/image.rb +31 -18
  47. data/lib/dcmgr/models/instance.rb +137 -143
  48. data/lib/dcmgr/models/instance_nic.rb +52 -29
  49. data/lib/dcmgr/models/instance_security_group.rb +9 -0
  50. data/lib/dcmgr/models/instance_spec.rb +163 -31
  51. data/lib/dcmgr/models/ip_lease.rb +10 -21
  52. data/lib/dcmgr/models/mac_lease.rb +30 -11
  53. data/lib/dcmgr/models/network.rb +148 -27
  54. data/lib/dcmgr/models/physical_network.rb +18 -0
  55. data/lib/dcmgr/models/quota.rb +0 -10
  56. data/lib/dcmgr/models/request_log.rb +3 -18
  57. data/lib/dcmgr/models/security_group.rb +66 -0
  58. data/lib/dcmgr/models/security_group_rule.rb +145 -0
  59. data/lib/dcmgr/models/ssh_key_pair.rb +16 -19
  60. data/lib/dcmgr/models/{storage_pool.rb → storage_node.rb} +35 -25
  61. data/lib/dcmgr/models/tag.rb +0 -14
  62. data/lib/dcmgr/models/tag_mapping.rb +1 -7
  63. data/lib/dcmgr/models/vlan_lease.rb +2 -8
  64. data/lib/dcmgr/models/volume.rb +49 -37
  65. data/lib/dcmgr/models/volume_snapshot.rb +15 -17
  66. data/lib/dcmgr/node_modules/hva_collector.rb +69 -28
  67. data/lib/dcmgr/node_modules/instance_ha.rb +23 -12
  68. data/lib/dcmgr/node_modules/instance_monitor.rb +16 -2
  69. data/lib/dcmgr/node_modules/openflow_controller.rb +784 -0
  70. data/lib/dcmgr/node_modules/scheduler.rb +189 -0
  71. data/lib/dcmgr/node_modules/service_netfilter.rb +452 -227
  72. data/lib/dcmgr/node_modules/service_openflow.rb +731 -0
  73. data/lib/dcmgr/node_modules/sta_collector.rb +20 -0
  74. data/lib/dcmgr/node_modules/sta_tgt_initializer.rb +35 -0
  75. data/lib/dcmgr/rack/request_logger.rb +11 -6
  76. data/lib/dcmgr/rpc/hva_handler.rb +256 -110
  77. data/lib/dcmgr/rpc/sta_handler.rb +244 -0
  78. data/lib/dcmgr/scheduler.rb +122 -8
  79. data/lib/dcmgr/scheduler/host_node/exclude_same.rb +24 -0
  80. data/lib/dcmgr/scheduler/host_node/find_first.rb +12 -0
  81. data/lib/dcmgr/scheduler/host_node/least_usage.rb +28 -0
  82. data/lib/dcmgr/scheduler/host_node/per_instance.rb +18 -0
  83. data/lib/dcmgr/scheduler/host_node/specify_node.rb +26 -0
  84. data/lib/dcmgr/scheduler/network/flat_single.rb +23 -0
  85. data/lib/dcmgr/scheduler/network/nat_one_to_one.rb +23 -0
  86. data/lib/dcmgr/scheduler/network/per_instance.rb +39 -0
  87. data/lib/dcmgr/scheduler/network/vif_template.rb +19 -0
  88. data/lib/dcmgr/scheduler/storage_node/find_first.rb +13 -0
  89. data/lib/dcmgr/scheduler/storage_node/least_usage.rb +23 -0
  90. data/lib/dcmgr/storage_service.rb +39 -40
  91. data/lib/dcmgr/tags.rb +3 -3
  92. data/lib/dcmgr/version.rb +1 -1
  93. data/lib/dcmgr/vnet.rb +105 -0
  94. data/lib/dcmgr/vnet/factories.rb +141 -0
  95. data/lib/dcmgr/vnet/isolators/by_securitygroup.rb +21 -0
  96. data/lib/dcmgr/vnet/isolators/dummy.rb +17 -0
  97. data/lib/dcmgr/vnet/netfilter/cache.rb +51 -0
  98. data/lib/dcmgr/vnet/netfilter/chain.rb +66 -0
  99. data/lib/dcmgr/vnet/netfilter/controller.rb +193 -0
  100. data/lib/dcmgr/vnet/netfilter/ebtables_rule.rb +53 -0
  101. data/lib/dcmgr/vnet/netfilter/iptables_rule.rb +45 -0
  102. data/lib/dcmgr/vnet/netfilter/task_manager.rb +459 -0
  103. data/lib/dcmgr/vnet/tasks/accept_all_dns.rb +19 -0
  104. data/lib/dcmgr/vnet/tasks/accept_arp_broadcast.rb +24 -0
  105. data/lib/dcmgr/vnet/tasks/accept_arp_from_friends.rb +34 -0
  106. data/lib/dcmgr/vnet/tasks/accept_arp_from_gateway.rb +21 -0
  107. data/lib/dcmgr/vnet/tasks/accept_arp_to_host.rb +30 -0
  108. data/lib/dcmgr/vnet/tasks/accept_ip_from_friends.rb +26 -0
  109. data/lib/dcmgr/vnet/tasks/accept_ip_from_gateway.rb +23 -0
  110. data/lib/dcmgr/vnet/tasks/accept_ip_to_anywhere.rb +18 -0
  111. data/lib/dcmgr/vnet/tasks/accept_related_established.rb +45 -0
  112. data/lib/dcmgr/vnet/tasks/accept_wakame_dhcp_only.rb +33 -0
  113. data/lib/dcmgr/vnet/tasks/accept_wakame_dns_only.rb +33 -0
  114. data/lib/dcmgr/vnet/tasks/debug_iptables.rb +21 -0
  115. data/lib/dcmgr/vnet/tasks/drop_arp_forwarding.rb +27 -0
  116. data/lib/dcmgr/vnet/tasks/drop_arp_to_host.rb +24 -0
  117. data/lib/dcmgr/vnet/tasks/drop_ip_from_anywhere.rb +18 -0
  118. data/lib/dcmgr/vnet/tasks/drop_ip_spoofing.rb +34 -0
  119. data/lib/dcmgr/vnet/tasks/drop_mac_spoofing.rb +33 -0
  120. data/lib/dcmgr/vnet/tasks/exclude_from_nat.rb +47 -0
  121. data/lib/dcmgr/vnet/tasks/security_group.rb +37 -0
  122. data/lib/dcmgr/vnet/tasks/static_nat.rb +54 -0
  123. data/lib/dcmgr/vnet/tasks/translate_metadata_address.rb +32 -0
  124. data/web/metadata/config.ru +1 -1
  125. metadata +174 -89
  126. data/lib/dcmgr/cli/group.rb +0 -101
  127. data/lib/dcmgr/endpoints/core_api_mock.rb +0 -865
  128. data/lib/dcmgr/models/host_pool.rb +0 -122
  129. data/lib/dcmgr/models/instance_netfilter_group.rb +0 -16
  130. data/lib/dcmgr/models/netfilter_group.rb +0 -89
  131. data/lib/dcmgr/models/netfilter_rule.rb +0 -21
  132. data/lib/dcmgr/scheduler/find_last.rb +0 -16
  133. data/lib/dcmgr/scheduler/find_random.rb +0 -16
  134. data/lib/dcmgr/stm/instance.rb +0 -25
  135. data/lib/dcmgr/stm/snapshot_context.rb +0 -33
  136. data/lib/dcmgr/stm/volume_context.rb +0 -65
@@ -1,122 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
-
3
- require 'isono'
4
-
5
- module Dcmgr::Models
6
- class HostPool < AccountResource
7
- taggable 'hp'
8
- with_timestamps
9
-
10
- HYPERVISOR_XEN_34='xen-3.4'
11
- HYPERVISOR_XEN_40='xen-4.0'
12
- HYPERVISOR_KVM='kvm'
13
-
14
- ARCH_X86=:x86.to_s
15
- ARCH_X86_64=:x86_64.to_s
16
-
17
- SUPPORTED_ARCH=[ARCH_X86, ARCH_X86_64]
18
- SUPPORTED_HYPERVISOR=[HYPERVISOR_KVM]
19
-
20
- inheritable_schema do
21
- String :node_id, :size=>80, :null=>true
22
-
23
- String :arch, :size=>10, :null=>false # :x86, :x86_64
24
- String :hypervisor, :size=>30, :null=>false
25
-
26
- Fixnum :offering_cpu_cores, :null=>false, :unsigned=>true
27
- Fixnum :offering_memory_size, :null=>false, :unsigned=>true
28
-
29
- index :node_id
30
- end
31
-
32
- one_to_many :instances
33
- many_to_one :node, :class=>Isono::Models::NodeState, :key=>:node_id, :primary_key=>:node_id
34
-
35
- def after_initialize
36
- super
37
- end
38
-
39
- def validate
40
- super
41
- # for compatibility: hva.xxx or hva-xxxx
42
- unless self.node_id =~ /^hva[-.]/
43
- errors.add(:node_id, "is invalid ID: #{self.node_id}")
44
- end
45
-
46
- if (h = self.class.filter(:node_id=>self.node_id).first) && h.id != self.id
47
- errors.add(:node_id, " #{self.node_id} is already been associated to #{h.canonical_uuid} ")
48
- end
49
-
50
- unless SUPPORTED_ARCH.member?(self.arch)
51
- errors.add(:arch, "unknown architecture type: #{self.arch}")
52
- end
53
-
54
- unless self.offering_cpu_cores > 0
55
- errors.add(:offering_cpu_cores, "it must have digit more than zero")
56
- end
57
- unless self.offering_memory_size > 0
58
- errors.add(:offering_memory_size, "it must have digit more than zero")
59
- end
60
- end
61
-
62
- def to_hash
63
- super.merge(:status=>self.status)
64
- end
65
-
66
- # Check if the resources exist depending on the HostPool.
67
- # @return [boolean]
68
- def depend_resources?
69
- !self.instances_dataset.runnings.empty?
70
- end
71
-
72
- # Factory method for Instance model to run on this HostPool.
73
- # @param [Models::Account] account
74
- # @param [Models::Image] image
75
- # @param [Models::InstanceSpec] spec
76
- # @param [Models::Network] network
77
- # @return [Models::Instance] created new Instance object.
78
- def create_instance(account, image, spec, network, &blk)
79
- raise ArgumentError unless image.is_a?(Image)
80
- raise ArgumentError unless spec.is_a?(InstanceSpec)
81
- raise ArgumentError unless network.is_a?(Network)
82
- i = Instance.new &blk
83
- i.account_id = account.canonical_uuid
84
- i.image = image
85
- i.instance_spec = spec
86
- i.cpu_cores = spec.cpu_cores
87
- i.memory_size = spec.memory_size
88
- i.quota_weight = spec.quota_weight
89
- i.host_pool = self
90
- i.save
91
-
92
- vnic = i.add_nic(network)
93
- IpLease.lease(vnic, network)
94
-
95
- #Lease the nat ip in case there is an outside network mapped
96
- nat_network = Network.find(:id => vnic[:nat_network_id])
97
- IpLease.lease(vnic,nat_network) unless nat_network.nil?
98
- i
99
- end
100
-
101
- def status
102
- node.nil? ? :offline : node.state
103
- end
104
-
105
- # Returns true/false if the host pool has enough capacity to run the spec.
106
- # @param [InstanceSpec] spec
107
- def check_capacity(spec)
108
- raise TypeError unless spec.is_a?(InstanceSpec)
109
- inst_on_hp = self.instances_dataset.lives.all
110
-
111
- (self.offering_cpu_cores >= inst_on_hp.inject(0) {|t, i| t += i.cpu_cores } + spec.cpu_cores) &&
112
- (self.offering_memory_size >= inst_on_hp.inject(0) {|t, i| t += i.memory_size } + spec.memory_size)
113
- end
114
-
115
- def to_api_document
116
- h = to_hash
117
- h.delete(:node_id)
118
- h
119
- end
120
-
121
- end
122
- end
@@ -1,16 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
-
3
- module Dcmgr::Models
4
- class InstanceNetfilterGroup < BaseNew
5
-
6
- inheritable_schema do
7
- Fixnum :instance_id, :null=>false
8
- Fixnum :netfilter_group_id, :null=>false
9
- end
10
- # with_timestamps
11
-
12
- many_to_one :instance
13
- many_to_one :netfilter_group
14
- end
15
-
16
- end
@@ -1,89 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
-
3
- module Dcmgr::Models
4
- class NetfilterGroup < AccountResource
5
- taggable 'ng'
6
- with_timestamps
7
-
8
- inheritable_schema do
9
- String :name, :null=>false
10
- String :description
11
- Text :rule
12
- index [:account_id, :name], {:unique=>true}
13
- end
14
-
15
- one_to_many :netfilter_rules
16
- many_to_many :instances,:join_table => :instance_netfilter_groups
17
-
18
- def to_hash
19
- h = super
20
- h = h.merge({
21
- :rule => rule.to_s,
22
- :rules => netfilter_rules.map { |rule| rule.to_hash },
23
- })
24
- #{
25
- #:id => self.canonical_uuid,
26
- #:name => name,
27
- #:description => description,
28
- #:rules => netfilter_rules.map { |rule| rule.to_hash },
29
- #}
30
- end
31
-
32
- def to_api_document
33
- to_hash
34
- end
35
-
36
- def to_tiny_hash
37
- {
38
- :name => self.name,
39
- :uuid => self.canonical_uuid,
40
- }
41
- end
42
-
43
- def self.create_group(account_id, params)
44
- grp = self.create(:account_id => account_id,
45
- :name => params[:name],
46
- :rule => params[:rule],
47
- :description => params[:description])
48
- grp.build_rule
49
- grp
50
- end
51
-
52
- def flush_rule
53
- NetfilterRule.filter(:netfilter_group_id => self.id).destroy
54
- end
55
-
56
- def before_destroy
57
- return false if self.instances.size > 0
58
-
59
- self.flush_rule
60
- super
61
- end
62
- alias :destroy_group :destroy
63
-
64
- def rebuild_rule
65
- self.flush_rule
66
- self.build_rule
67
- end
68
-
69
- def build_rule
70
- return if self.rule.nil?
71
-
72
- self.rule.split("\n").each { |permission|
73
- # [ToDo]
74
- # to make strong parser
75
- next if permission =~ /\A#/
76
- next if permission.length == 0
77
-
78
- # [format] protocol,source,destination
79
- # - protocol: tcp|udp|icmp
80
- # - source: IPAddr|CIDR|Owner:Group
81
- # - destination: port|icmp-type
82
- NetfilterRule.create(:netfilter_group_id => self.id,
83
- :permission => permission)
84
-
85
- }
86
- end
87
-
88
- end
89
- end
@@ -1,21 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
-
3
- module Dcmgr::Models
4
- class NetfilterRule < BaseNew
5
- with_timestamps
6
-
7
- inheritable_schema do
8
- Fixnum :netfilter_group_id, :null=>false
9
- String :permission, :null=>false
10
- end
11
-
12
- many_to_one :netfilter_group
13
-
14
- def to_hash
15
- {
16
- :permission => permission,
17
- }
18
- end
19
-
20
- end
21
- end
@@ -1,16 +0,0 @@
1
-
2
- module Dcmgr
3
- module PhysicalHostScheduler
4
- # This is a simple host scheduler which gets back a host found
5
- # at the top of hosts list.
6
- class FindLast
7
- def assign_to_instance(hosts, instance)
8
- Dcmgr::logger.debug "assign to instance (%d hosts)" % hosts.length
9
- p 'hosts'
10
- p hosts
11
- return hosts.last
12
- raise NoPhysicalHostError.new("can't assign physical host")
13
- end
14
- end
15
- end
16
- end
@@ -1,16 +0,0 @@
1
-
2
- module Dcmgr
3
- module PhysicalHostScheduler
4
- # This is a simple host scheduler which gets back a host found
5
- # at the top of hosts list.
6
- class FindRandom
7
- def assign_to_instance(hosts, instance)
8
- Dcmgr::logger.debug "assign to instance (%d hosts)" % hosts.length
9
- hvc_host = hosts[ rand(hosts.length) ]
10
- p 'hvc_host'
11
- p hvc_host
12
- raise NoPhysicalHostError.new("can't assign physical host")
13
- end
14
- end
15
- end
16
- end
@@ -1,25 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
-
3
- require 'statemachine'
4
-
5
- module Dcmgr::Stm
6
- class Instance
7
- STM = Statemachine.build {
8
- startstate :pending
9
- superstate :instance_condition do
10
- trans :pending, :on_create, :starting
11
- trans :starting, :on_started, :running
12
- trans :running, :on_shutdown, :shuttingdown
13
- trans :shuttingdown, :on_terminated, :terminated
14
-
15
- event :on_fail, :failed
16
- end
17
-
18
- trans :failed, :on_fail, :failed
19
- }
20
-
21
- def initialize
22
- end
23
-
24
- end
25
- end
@@ -1,33 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- require 'statemachine'
3
-
4
- module Dcmgr::Stm
5
- class SnapshotContext < OpenStruct
6
-
7
- attr_reader :stm
8
-
9
- def initialize(snapshot_id=nil)
10
- super({:snapshot_id=>snapshot_id})
11
- @stm = Statemachine.build {
12
- trans :registering, :on_create, :creating
13
- trans :creating, :on_create, :available
14
- trans :available, :on_delete, :deleting
15
- trans :deleting, :on_delete, :deleted
16
-
17
- trans :registering, :on_fail, :failed
18
- trans :creating, :on_fail, :failed
19
- trans :available, :on_fail, :failed
20
- trans :deleting, :on_fail, :failed
21
- }
22
- @stm.context = self
23
- end
24
-
25
- def state
26
- @stm.state
27
- end
28
-
29
- def to_hash
30
- @table.dup.merge({:state=>@stm.state})
31
- end
32
- end
33
- end
@@ -1,65 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- require 'statemachine'
3
-
4
- module Dcmgr::Stm
5
- class VolumeContext < OpenStruct
6
- attr_reader :stm
7
-
8
- def initialize(volume_id=nil)
9
- super({:volume_id => volume_id,
10
- :export_path => nil,
11
- :transport_information => nil,
12
- :instance_id => nil,
13
- :host_device_name => nil,
14
- :guest_device_name => nil,
15
- :deleted_at => nil,
16
- :attached_at => nil,
17
- :detached_at => nil,
18
- })
19
- @stm = Statemachine.build {
20
- startstate :registering
21
- superstate :volume_condition do
22
- trans :registering, :on_create, :creating
23
- trans :creating, :on_register, :available
24
- trans :available, :on_attach, :attaching
25
- trans :attaching, :on_attach, :attached
26
- trans :attached, :on_detach, :detaching
27
- trans :detaching, :on_detach, :available
28
-
29
- event :on_fail, :failed
30
- event :on_deregister, :deregistering
31
- end
32
-
33
- trans :failed, :on_create, :creating
34
- trans :failed, :on_register, :available
35
- trans :failed, :on_fail, :failed
36
- trans :failed, :on_deregister, :deleting
37
- trans :failed, :on_delete, :deleted
38
- trans :deregistering, :on_delete, :deleting
39
- trans :deleting, :on_delete, :deleted
40
- trans :deleted, :on_delete, :deleted
41
- }
42
- @stm.context = self
43
- end
44
-
45
- def state
46
- @stm.state
47
- end
48
-
49
- def to_hash(hash={})
50
- @table.dup.merge({:state=>@stm.state}).merge(hash)
51
- end
52
-
53
- def on_delete
54
- self.deleted_at = Time.now
55
- end
56
-
57
- def on_attach
58
- self.attached_at = Time.now
59
- end
60
-
61
- def on_detach
62
- self.detached_at = Time.now
63
- end
64
- end
65
- end