deltacloud-core 1.0.5 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (141) hide show
  1. data/Rakefile +10 -2
  2. data/bin/deltacloudd +10 -10
  3. data/config.ru +2 -1
  4. data/config/drivers/digitalocean.yaml +3 -0
  5. data/deltacloud-core.gemspec +13 -6
  6. data/lib/cimi/collections.rb +1 -1
  7. data/lib/cimi/collections/address_templates.rb +27 -3
  8. data/lib/cimi/collections/addresses.rb +1 -1
  9. data/lib/cimi/collections/base.rb +1 -0
  10. data/lib/cimi/collections/cloud_entry_point.rb +4 -0
  11. data/lib/cimi/collections/credentials.rb +2 -2
  12. data/lib/cimi/collections/machine_images.rb +20 -0
  13. data/lib/cimi/collections/machine_templates.rb +72 -0
  14. data/lib/cimi/collections/machines.rb +50 -41
  15. data/lib/cimi/collections/network_ports.rb +3 -3
  16. data/lib/cimi/collections/networks.rb +4 -4
  17. data/lib/cimi/collections/resource_metadata.rb +1 -1
  18. data/lib/cimi/collections/volume_configurations.rb +25 -0
  19. data/lib/cimi/collections/volume_images.rb +21 -1
  20. data/lib/cimi/collections/volume_templates.rb +69 -0
  21. data/lib/cimi/collections/volumes.rb +5 -10
  22. data/lib/cimi/dependencies.rb +0 -1
  23. data/lib/cimi/helpers.rb +4 -1
  24. data/lib/cimi/helpers/cimi_helper.rb +62 -0
  25. data/lib/cimi/helpers/database_helper.rb +95 -0
  26. data/lib/cimi/models.rb +15 -1
  27. data/lib/cimi/models/address.rb +10 -5
  28. data/lib/cimi/models/address_template.rb +67 -3
  29. data/lib/cimi/models/base.rb +8 -5
  30. data/lib/cimi/models/cloud_entry_point.rb +6 -1
  31. data/lib/cimi/models/collection.rb +9 -4
  32. data/lib/cimi/models/disk.rb +6 -1
  33. data/lib/cimi/models/errors.rb +8 -0
  34. data/lib/cimi/models/machine.rb +68 -42
  35. data/lib/cimi/models/machine_configuration.rb +2 -2
  36. data/lib/cimi/models/machine_image.rb +41 -6
  37. data/lib/cimi/models/machine_template.rb +58 -0
  38. data/lib/cimi/models/machine_volume.rb +51 -3
  39. data/lib/cimi/models/resource_metadata.rb +88 -25
  40. data/lib/cimi/models/schema.rb +19 -5
  41. data/lib/cimi/models/volume.rb +66 -30
  42. data/lib/cimi/models/volume_configuration.rb +53 -21
  43. data/lib/cimi/models/volume_image.rb +24 -9
  44. data/lib/cimi/models/volume_template.rb +61 -0
  45. data/lib/cimi/server.rb +0 -6
  46. data/lib/db.rb +82 -0
  47. data/lib/db/address_template.rb +15 -0
  48. data/lib/db/entity.rb +22 -0
  49. data/lib/db/machine_template.rb +10 -0
  50. data/lib/db/provider.rb +13 -0
  51. data/lib/db/volume_configuration.rb +10 -0
  52. data/lib/db/volume_template.rb +10 -0
  53. data/lib/deltacloud/collections/addresses.rb +1 -1
  54. data/lib/deltacloud/collections/base.rb +12 -1
  55. data/lib/deltacloud/collections/buckets.rb +41 -8
  56. data/lib/deltacloud/collections/drivers.rb +1 -1
  57. data/lib/deltacloud/collections/firewalls.rb +2 -2
  58. data/lib/deltacloud/collections/images.rb +1 -1
  59. data/lib/deltacloud/collections/instances.rb +7 -1
  60. data/lib/deltacloud/collections/keys.rb +1 -1
  61. data/lib/deltacloud/collections/load_balancers.rb +4 -4
  62. data/lib/deltacloud/collections/storage_snapshots.rb +5 -1
  63. data/lib/deltacloud/collections/storage_volumes.rb +7 -3
  64. data/lib/deltacloud/drivers/base_driver.rb +10 -9
  65. data/lib/deltacloud/drivers/cimi_features.rb +42 -0
  66. data/lib/deltacloud/drivers/digitalocean/digitalocean_driver.rb +307 -0
  67. data/lib/deltacloud/drivers/ec2/ec2_driver.rb +40 -14
  68. data/lib/deltacloud/drivers/exceptions.rb +8 -0
  69. data/lib/deltacloud/drivers/features.rb +19 -2
  70. data/lib/deltacloud/drivers/fgcp/fgcp_client.rb +11 -0
  71. data/lib/deltacloud/drivers/fgcp/fgcp_driver.rb +83 -11
  72. data/lib/deltacloud/drivers/gogrid/gogrid_client.rb +1 -1
  73. data/lib/deltacloud/drivers/mock/mock_client.rb +2 -4
  74. data/lib/deltacloud/drivers/mock/mock_driver.rb +29 -0
  75. data/lib/deltacloud/drivers/openstack/openstack_driver.rb +153 -36
  76. data/lib/deltacloud/drivers/rackspace/anti_cache_monkey_patch.rb +20 -0
  77. data/lib/deltacloud/drivers/rackspace/rackspace_driver.rb +1 -0
  78. data/lib/deltacloud/drivers/rhevm/rhevm_driver.rb +30 -12
  79. data/lib/deltacloud/drivers/sbc/sbc_client.rb +0 -1
  80. data/lib/deltacloud/drivers/sbc/sbc_driver.rb +5 -1
  81. data/lib/deltacloud/drivers/vsphere/vsphere_client.rb +1 -1
  82. data/lib/deltacloud/drivers/vsphere/vsphere_driver.rb +19 -9
  83. data/lib/deltacloud/helpers/blob_stream_helper.rb +42 -3
  84. data/lib/deltacloud/helpers/deltacloud_helper.rb +31 -14
  85. data/lib/deltacloud/models/address.rb +9 -0
  86. data/lib/deltacloud/models/base_model.rb +4 -0
  87. data/lib/deltacloud/models/blob.rb +12 -0
  88. data/lib/deltacloud/models/bucket.rb +9 -0
  89. data/lib/deltacloud/models/firewall.rb +13 -1
  90. data/lib/deltacloud/models/firewall_rule.rb +14 -0
  91. data/lib/deltacloud/models/hardware_profile.rb +14 -0
  92. data/lib/deltacloud/models/image.rb +15 -0
  93. data/lib/deltacloud/models/instance.rb +40 -1
  94. data/lib/deltacloud/models/instance_address.rb +9 -0
  95. data/lib/deltacloud/models/key.rb +15 -0
  96. data/lib/deltacloud/models/load_balancer.rb +20 -0
  97. data/lib/deltacloud/models/metric.rb +15 -0
  98. data/lib/deltacloud/models/provider.rb +8 -0
  99. data/lib/deltacloud/models/realm.rb +9 -0
  100. data/lib/deltacloud/models/state_machine.rb +8 -0
  101. data/lib/deltacloud/models/storage_snapshot.rb +11 -0
  102. data/lib/deltacloud/models/storage_volume.rb +24 -0
  103. data/lib/deltacloud/server.rb +1 -3
  104. data/lib/deltacloud/version.rb +2 -1
  105. data/lib/deltacloud_rack.rb +1 -1
  106. data/tests/cimi/db/database_helper_test.rb +190 -0
  107. data/tests/cimi/db/db_helper.rb +30 -0
  108. data/tests/cimi/db/schema_test.rb +94 -0
  109. data/tests/cimi/model/collection_spec.rb +0 -1
  110. data/tests/cimi/model/machine_spec.rb +17 -0
  111. data/tests/cimi/spec_helper.rb +3 -2
  112. data/tests/deltacloud/collections/buckets_collection_test.rb +3 -0
  113. data/tests/deltacloud/collections/drivers_collection_test.rb +10 -0
  114. data/tests/deltacloud/collections/hardware_profiles_collection_test.rb +4 -0
  115. data/tests/deltacloud/collections/images_collection_test.rb +4 -0
  116. data/tests/deltacloud/collections/instances_collection_test.rb +14 -1
  117. data/tests/deltacloud/collections/keys_collection_test.rb +4 -0
  118. data/tests/deltacloud/collections/realms_collection_test.rb +47 -0
  119. data/tests/deltacloud/collections/storage_snapshots_collection_test.rb +47 -0
  120. data/tests/deltacloud/collections/storage_volumes_collection_test.rb +47 -0
  121. data/tests/deltacloud/common.rb +15 -0
  122. data/tests/deltacloud/deltacloud_helper_test.rb +0 -4
  123. data/tests/deltacloud/launcher_test.rb +108 -0
  124. data/tests/drivers/ec2/buckets_test.rb +2 -1
  125. data/tests/drivers/mock/buckets_test.rb +27 -0
  126. data/tests/drivers/mock/instances_test.rb +6 -0
  127. data/tests/drivers/openstack/common.rb +1 -1
  128. data/tests/drivers/openstack/hardware_profiles_test.rb +2 -1
  129. data/tests/drivers/openstack/instances_test.rb +18 -17
  130. data/tests/drivers/rhevm/common.rb +1 -0
  131. data/tests/drivers/rhevm/images_test.rb +1 -1
  132. data/tests/drivers/rhevm/instance_test.rb +7 -7
  133. data/tests/helpers/rack/rack_matrix_params_test.rb +0 -2
  134. data/tests/test_helper.rb +2 -1
  135. data/views/errors/403.xml.haml +6 -0
  136. data/views/errors/409.html.haml +47 -0
  137. data/views/errors/409.xml.haml +11 -0
  138. data/views/errors/502.html.haml +6 -5
  139. data/views/images/show.xml.haml +3 -2
  140. data/views/instances/new.html.haml +1 -1
  141. metadata +77 -30
@@ -507,33 +507,51 @@ module Deltacloud
507
507
  #--
508
508
  # Create Blob - NON Streaming way (i.e. was called with POST html multipart form data)
509
509
  #--
510
+ #also called for segmented blobs - as final call with blob manifest
510
511
  def create_blob(credentials, bucket_id, blob_id, data = nil, opts = {})
511
512
  s3_client = new_client(credentials, :s3)
512
513
  #data is a construct with the temporary file created by server @.tempfile
513
514
  #also file[:type] will give us the content-type
514
- res = nil
515
- # File stream needs to be reopened in binary mode for whatever reason
516
- file = File::open(data[:tempfile].path, 'rb')
517
- #insert ec2-specific header for user metadata ... x-amz-meta-KEY = VALUE
518
- BlobHelper::rename_metadata_headers(opts, 'x-amz-meta-')
519
- opts["Content-Type"] = data[:type]
520
- safely do
521
- res = s3_client.interface.put(bucket_id,
522
- blob_id,
523
- file,
524
- opts)
515
+ if(opts[:segment_manifest])
516
+ safely do
517
+ s3_client.interface.complete_multipart(bucket_id, blob_id, opts[:segmented_blob_id], opts[:segment_manifest])
518
+ end
519
+ else
520
+ # File stream needs to be reopened in binary mode
521
+ file = File::open(data[:tempfile].path, 'rb')
522
+ #insert ec2-specific header for user metadata ... x-amz-meta-KEY = VALUE
523
+ BlobHelper::rename_metadata_headers(opts, 'x-amz-meta-')
524
+ opts["Content-Type"] = data[:type]
525
+ safely do
526
+ s3_client.interface.put(bucket_id,
527
+ blob_id,
528
+ file,
529
+ opts)
530
+ end
525
531
  end
526
532
  #create a new Blob object and return that
527
533
  Blob.new( { :id => blob_id,
528
534
  :bucket => bucket_id,
529
- :content_length => data[:tempfile].length,
530
- :content_type => data[:type],
535
+ :content_length => ((data && data[:tempfile]) ? data[:tempfile].length : nil),
536
+ :content_type => ((data && data[:type]) ? data[:type] : nil),
531
537
  :last_modified => '',
532
538
  :user_metadata => opts.select{|k,v| k.match(/^x-amz-meta-/i)}
533
539
  }
534
540
  )
535
541
  end
536
542
 
543
+ def init_segmented_blob(credentials, opts={})
544
+ s3_client = new_client(credentials, :s3)
545
+ safely do
546
+ s3_client.interface.initiate_multipart(opts[:bucket],opts[:id])
547
+ end
548
+
549
+ end
550
+
551
+ def blob_segment_id(request, response)
552
+ response["etag"].gsub("\"", "")
553
+ end
554
+
537
555
  #--
538
556
  # Delete Blob
539
557
  #--
@@ -591,8 +609,16 @@ module Deltacloud
591
609
  timestamp = Time.now.httpdate
592
610
  string_to_sign =
593
611
  "PUT\n\n#{params[:content_type]}\n#{timestamp}\n#{signature_meta_string}/#{params[:bucket]}/#{params[:blob]}"
612
+ if BlobHelper.segmented_blob_op_type(params[:context]) == "segment"
613
+ partNumber = BlobHelper.segment_order(params[:context])
614
+ uploadId = BlobHelper.segmented_blob_id(params[:context])
615
+ segment_string = "?partNumber=#{partNumber}&uploadId=#{uploadId}"
616
+ string_to_sign << segment_string
617
+ request = Net::HTTP::Put.new("/#{params[:blob]}#{segment_string}")
618
+ else
619
+ request = Net::HTTP::Put.new("/#{params[:blob]}")
620
+ end
594
621
  auth_string = Aws::Utils::sign(params[:password], string_to_sign)
595
- request = Net::HTTP::Put.new("/#{params[:blob]}")
596
622
  request['Host'] = "#{params[:bucket]}.#{uri.host}"
597
623
  request['Date'] = timestamp
598
624
  request['Content-Type'] = params[:content_type]
@@ -58,6 +58,13 @@ module Deltacloud
58
58
  end
59
59
  end
60
60
 
61
+ class Conflict < DeltacloudException
62
+ def initialize(e, message=nil)
63
+ message ||= e.message
64
+ super(409, e.class.name, message, e.backtrace)
65
+ end
66
+ end
67
+
61
68
  class MethodNotAllowed < DeltacloudException
62
69
  def initialize(e, message=nil)
63
70
  message ||= e.message
@@ -157,6 +164,7 @@ module Deltacloud
157
164
  when 406 then UnknownMediaTypeError.new(e, @message)
158
165
  when 405 then MethodNotAllowed.new(e, @message)
159
166
  when 400 then ValidationFailure.new(e, @message)
167
+ when 409 then Conflict.new(e, @message)
160
168
  when 500 then BackendError.new(e, @message)
161
169
  when 501 then NotImplemented.new(e, @message)
162
170
  when 502 then ProviderError.new(e, @message)
@@ -112,8 +112,6 @@ module Deltacloud
112
112
  operation :create do
113
113
  param :keyname, :string, :optional, [], "Key authentification method"
114
114
  end
115
- operation :show do
116
- end
117
115
  end
118
116
 
119
117
  feature :authentication_password, :for => :instances do
@@ -163,6 +161,25 @@ module Deltacloud
163
161
  end
164
162
  end
165
163
 
164
+ feature :volume_name, :for => :storage_volumes do
165
+ description "Specify name when creating storage_volume"
166
+ operation :create do
167
+ param :name, :string, :optional
168
+ end
169
+ end
170
+
171
+ feature :volume_description, :for => :storage_volumes do
172
+ description "Specify description when creating storage_volume"
173
+ operation :create do
174
+ param :description, :string, :optional
175
+ end
176
+ end
177
+
178
+
179
+
180
+
181
+
182
+
166
183
  end
167
184
 
168
185
  end
@@ -115,6 +115,17 @@ class FgcpClient
115
115
  'networkId' => network_id})
116
116
  end
117
117
 
118
+ def create_vservers(vsys_id, vservers_xml)
119
+ @version = '2012-07-20'
120
+ request('CreateMultipleVServer',
121
+ {
122
+ 'vsysId' => vsys_id,
123
+ },
124
+ vservers_xml,
125
+ 'vserversXMLFilePath'
126
+ )
127
+ end
128
+
118
129
  def destroy_vserver(vserver_id)
119
130
  request('DestroyVServer', {'vsysId' => extract_vsys_id(vserver_id), 'vserverId' => vserver_id})
120
131
  end
@@ -30,6 +30,7 @@ class FgcpDriver < Deltacloud::BaseDriver
30
30
  feature :instances, :user_name
31
31
  feature :instances, :metrics
32
32
  feature :instances, :realm_filter
33
+ feature :instances, :instance_count
33
34
  feature :images, :user_name
34
35
  feature :images, :user_description
35
36
 
@@ -343,7 +344,7 @@ class FgcpDriver < Deltacloud::BaseDriver
343
344
  # Create a new instance, given an image id
344
345
  # opts can include an optional name for the instance, hardware profile (hwp_id) and realm_id
345
346
  def create_instance(credentials, image_id, opts={})
346
- name = opts[:name]
347
+ name = (opts[:name] && opts[:name].length > 0)? opts[:name] : "server_#{Time.now.to_s}"
347
348
  # default to 'economy' or obtain latest hardware profiles and pick the lowest spec profile?
348
349
  hwp = opts[:hwp_id] || 'economy'
349
350
  network_id = opts[:realm_id]
@@ -355,9 +356,33 @@ class FgcpDriver < Deltacloud::BaseDriver
355
356
  # use first returned system's DMZ as realm
356
357
  network_id = xml ? xml[0]['vsys'][0]['vsysId'][0] + '-N-DMZ' : nil
357
358
  end
358
- xml = client.create_vserver(name, hwp, image_id, network_id)
359
- # returns vserver details
360
- instances(credentials, {:id => xml['vserverId'][0]}).first
359
+ if opts[:instance_count] and opts[:instance_count].to_i > 1
360
+
361
+ vservers = Array.new(opts[:instance_count].to_i) { |n|
362
+ {
363
+ 'vserverName' => "#{name}_#{n+1}",
364
+ 'vserverType' => hwp,
365
+ 'diskImageId' => image_id,
366
+ 'networkId' => network_id
367
+ }
368
+ }
369
+ new_vservers = { 'vservers' => { 'vserver' => vservers } }
370
+ vservers_xml = XmlSimple.xml_out(new_vservers,
371
+ 'RootName' => 'Request',
372
+ 'NoAttr' => true
373
+ )
374
+
375
+ xml = client.create_vservers(client.extract_vsys_id(network_id), vservers_xml)
376
+ vserver_ids = xml['vservers'][0]['vserver'].collect { |vserver| vserver['vserverId'][0] }
377
+ # returns vservers' details using filter
378
+ instances(credentials, {:realm_id => network_id}).select { |instance|
379
+ vserver_ids.include? instance.id
380
+ }
381
+ else
382
+ xml = client.create_vserver(name, hwp, image_id, network_id)
383
+ # returns vserver details
384
+ instances(credentials, {:id => xml['vserverId'][0]}).first
385
+ end
361
386
  end
362
387
  end
363
388
 
@@ -697,6 +722,7 @@ class FgcpDriver < Deltacloud::BaseDriver
697
722
  addresses
698
723
  end
699
724
 
725
+ # allocates (and enables) new ip in specified vsys/network
700
726
  def create_address(credentials, opts={})
701
727
  safely do
702
728
  client = new_client(credentials)
@@ -710,10 +736,32 @@ class FgcpDriver < Deltacloud::BaseDriver
710
736
  opts[:realm_id] = xml[0]['vsys'][0]['vsysId'][0] if xml
711
737
  end
712
738
 
739
+ old_ips = []
740
+ xml = client.list_public_ips(opts[:realm_id])['publicips']
741
+ old_ips = xml[0]['publicip'].collect { |ip| ip['address'][0]} if xml and xml[0]['publicip']
742
+
713
743
  client.allocate_public_ip(opts[:realm_id])
744
+ # new address not returned immediately:
745
+ # Seems to take 15-30s. to appear in list, so poll for a while
746
+ # prepare dummy id in case new ip does not appear soon.
747
+ id = 'PENDING-xxx.xxx.xxx.xxx'
748
+ sleep(8)
749
+ 10.times {
750
+
751
+ sleep(5)
752
+ xml = client.list_public_ips(opts[:realm_id])['publicips']
753
+ if xml and xml[0]['publicip'] and xml[0]['publicip'].size > old_ips.size
754
+
755
+ new_ips = xml[0]['publicip'].collect { |ip| ip['address'][0]}
756
+ new_ip = (new_ips - old_ips).first
757
+ # enable IP address
758
+ client.attach_public_ip(opts[:realm_id], new_ip)
759
+ id = new_ip
760
+ break
761
+ end
762
+ }
763
+ Address.new(:id => id)
714
764
  end
715
- # new address not returned immediately!
716
- Address.new(:id => 'PENDING-xxx.xxx.xxx.xxx')
717
765
  end
718
766
 
719
767
  def destroy_address(credentials, opts={})
@@ -731,12 +779,23 @@ class FgcpDriver < Deltacloud::BaseDriver
731
779
  end
732
780
  end
733
781
  begin
734
- # detach just in case
782
+ # disable IP if still enabled
735
783
  client.detach_public_ip(opts[:realm_id], opts[:id])
784
+ sleep(8)
736
785
  rescue Exception => ex
737
786
  raise ex unless ex.message =~ /^ALREADY_DETACHED.*/
738
787
  end
739
- client.free_public_ip(opts[:realm_id], opts[:id])
788
+ attempts = 0
789
+ begin
790
+ # this may fail if the ip is still detaching, hence retry for a while
791
+ client.free_public_ip(opts[:realm_id], opts[:id])
792
+ rescue Exception => ex
793
+ raise unless attempts < 10 and ex.message =~ /^ILLEGAL_CONDITION.*/
794
+ # Detaching seems to take 15-30s, so keep trying for a while
795
+ sleep(5)
796
+ attempts += 1
797
+ retry
798
+ end
740
799
  end
741
800
  end
742
801
 
@@ -746,7 +805,9 @@ class FgcpDriver < Deltacloud::BaseDriver
746
805
  vsys_id = client.extract_vsys_id(opts[:instance_id])
747
806
 
748
807
  begin
808
+ # enable IP in case not enabled already
749
809
  client.attach_public_ip(vsys_id, opts[:id])
810
+ sleep(8)
750
811
  rescue Exception => ex
751
812
  raise ex unless ex.message =~ /^ALREADY_ATTACHED.*/
752
813
  end
@@ -768,7 +829,8 @@ class FgcpDriver < Deltacloud::BaseDriver
768
829
  fw_id = "#{vsys_id}-S-0001"
769
830
  nat_rules = client.get_efm_configuration(fw_id, 'FW_NAT_RULE')['efm'][0]['firewall'][0]['nat'][0]['rules'][0]
770
831
 
771
- if nat_rules and not nat_rules.empty? # happens only if no enabled IP address?
832
+ # TODO: if no IP address enabled yet
833
+ if nat_rules and not nat_rules.empty? and nat_rules['rule'].find { |rule| rule['publicIp'][0] == opts[:id] }
772
834
 
773
835
  nat_rules['rule'].each do |rule|
774
836
 
@@ -830,7 +892,6 @@ class FgcpDriver < Deltacloud::BaseDriver
830
892
  )
831
893
 
832
894
  client.update_efm_configuration(fw_id, 'FW_NAT_RULE', conf_xml_new)
833
- client.detach_public_ip(client.extract_vsys_id(opts[:realm_id]), opts[:id])
834
895
  end
835
896
  end
836
897
 
@@ -1194,8 +1255,11 @@ eofwopxml
1194
1255
  def create_load_balancer(credentials, opts={})
1195
1256
  safely do
1196
1257
  client = new_client(credentials)
1197
- # if opts['realm_id'].nil? network id specified, pick first vsys' DMZ?
1258
+ # if opts['realm_id'].nil? network id specified, pick first vsys' DMZ
1259
+ # if realm has SLB already, use that, else create
1198
1260
  # CreateEFM -vsysId vsysId -efmType SLB -efmName opts['name'] -networkId opts['realm_id']
1261
+ # if not started already, start
1262
+ # add group and return :id => efmId_groupId
1199
1263
  network_id = opts[:realm_id]
1200
1264
  if not network_id
1201
1265
  xml = client.list_vsys['vsyss']
@@ -1215,6 +1279,9 @@ eofwopxml
1215
1279
  def destroy_load_balancer(credentials, id)
1216
1280
  safely do
1217
1281
  client = new_client(credentials)
1282
+ # remove group from SLB
1283
+ # if no groups left, stop and destroy SLB
1284
+ # destroy in new thread? May fail if public IP associated?
1218
1285
  client.destroy_efm(id)
1219
1286
  end
1220
1287
  end
@@ -1376,6 +1443,11 @@ eofwopxml
1376
1443
  status 404 # Not Found
1377
1444
  end
1378
1445
 
1446
+ # reached maximum number of attempts while polling for an update
1447
+ on /Server did not include public IP address in FW NAT rules/ do
1448
+ status 504 # Gateway Timeout
1449
+ end
1450
+
1379
1451
  # wrong FW description (vsys descriptor)
1380
1452
  on /does not exist. Specify one of / do
1381
1453
  status 404 # Not Found
@@ -17,7 +17,7 @@
17
17
  require 'digest/md5'
18
18
  require 'cgi'
19
19
  require 'open-uri'
20
- require 'json'
20
+ require 'json/pure'
21
21
 
22
22
  module Kernel
23
23
  def suppress_warnings
@@ -25,11 +25,9 @@ module Deltacloud::Drivers::Mock
25
25
  def initialize(storage_root)
26
26
  @storage_root = storage_root
27
27
  @collections = []
28
-
29
- if ! File::directory?(@storage_root)
30
- FileUtils::rm_rf(@storage_root)
31
- FileUtils::mkdir_p(@storage_root, :mode => 0750)
28
+ if ! File::directory?(File::join(@storage_root, "images"))
32
29
  data = Dir[File::join(File::dirname(__FILE__), "data", "*")]
30
+ FileUtils::mkdir_p(@storage_root, :verbose => true)
33
31
  FileUtils::cp_r(data, @storage_root)
34
32
  end
35
33
  end
@@ -84,6 +84,15 @@ module Deltacloud::Drivers::Mock
84
84
  feature :images, :user_name
85
85
  feature :images, :user_description
86
86
 
87
+ #cimi features
88
+ feature :machines, :default_initial_state do
89
+ { :values => ["STARTED"] }
90
+ end
91
+ feature :machines, :initial_states do
92
+ { :values => ["STARTED", "STOPPED"]}
93
+ end
94
+
95
+
87
96
  def initialize
88
97
  if ENV["DELTACLOUD_MOCK_STORAGE"]
89
98
  storage_root = ENV["DELTACLOUD_MOCK_STORAGE"]
@@ -305,6 +314,26 @@ module Deltacloud::Drivers::Mock
305
314
  snapshots
306
315
  end
307
316
 
317
+ def create_storage_snapshot(credentials, opts={})
318
+ check_credentials(credentials)
319
+ id = "store_snapshot_#{Time.now.to_i}"
320
+ snapshot = {
321
+ :id => id,
322
+ :created => Time.now.to_s,
323
+ :state => "COMPLETED",
324
+ :storage_volume_id => opts[:volume_id],
325
+ }
326
+ snapshot.merge!({:name=>opts[:name]}) if opts[:name]
327
+ snapshot.merge!({:description=>opts[:description]}) if opts[:description]
328
+ @client.store(:storage_snapshots, snapshot)
329
+ StorageSnapshot.new(snapshot)
330
+ end
331
+
332
+ def destroy_storage_snapshot(credentials, opts={})
333
+ check_credentials(credentials)
334
+ @client.destroy(:storage_snapshots, opts[:id])
335
+ end
336
+
308
337
  def keys(credentials, opts={})
309
338
  check_credentials(credentials)
310
339
  result = @client.build_all(Key)
@@ -30,6 +30,8 @@ module Deltacloud
30
30
  feature :instances, :user_data
31
31
  feature :images, :user_name
32
32
  feature :keys, :import_key
33
+ feature :storage_volumes, :volume_name
34
+ feature :storage_volumes, :volume_description
33
35
 
34
36
  define_instance_states do
35
37
  start.to( :pending ) .on( :create )
@@ -38,6 +40,7 @@ module Deltacloud
38
40
  running.to( :stopping ) .on( :stop )
39
41
  stopping.to( :stopped ) .automatically
40
42
  stopped.to( :finish ) .automatically
43
+ error.from(:running, :pending, :stopping)
41
44
  end
42
45
 
43
46
  define_hardware_profile('default')
@@ -46,9 +49,14 @@ module Deltacloud
46
49
  #get the collections as defined by 'capability' and 'respond_to?' blocks
47
50
  super_collections = super
48
51
  begin
49
- new_client(credentials, :buckets)
52
+ new_client(credentials, "object-store")
50
53
  rescue Deltacloud::Exceptions::NotImplemented #OpenStack::Exception::NotImplemented...
51
- return super_collections - [Sinatra::Rabbit::BucketsCollection]
54
+ super_collections = super_collections - [Sinatra::Rabbit::BucketsCollection]
55
+ end
56
+ begin
57
+ new_client(credentials, "volume")
58
+ rescue Deltacloud::Exceptions::NotImplemented
59
+ super_collections = super_collections - [Sinatra::Rabbit::StorageVolumesCollection]
52
60
  end
53
61
  super_collections
54
62
  end
@@ -138,19 +146,19 @@ module Deltacloud
138
146
 
139
147
  def instances(credentials, opts={})
140
148
  os = new_client(credentials)
141
- insts = []
149
+ insts = attachments = []
142
150
  safely do
143
151
  if opts[:id]
144
152
  begin
145
153
  server = os.get_server(opts[:id])
146
- insts << convert_from_server(server, os.connection.authuser)
154
+ insts << convert_from_server(server, os.connection.authuser, get_attachments(opts[:id], os))
147
155
  rescue => e
148
156
  raise e unless e.message =~ /The resource could not be found/
149
157
  insts = []
150
158
  end
151
159
  else
152
160
  insts = os.list_servers_detail.collect do |s|
153
- convert_from_server(s, os.connection.authuser)
161
+ convert_from_server(s, os.connection.authuser,get_attachments(s[:id], os))
154
162
  end
155
163
  end
156
164
  end
@@ -173,13 +181,13 @@ module Deltacloud
173
181
  end
174
182
  if opts[:keyname] && opts[:keyname].length > 0
175
183
  params[:key_name]=opts[:keyname]
176
- end
177
- if opts[:user_data] && opts[:user_data].length > 0
178
- params[:user_data]=Base64.encode64(opts[:user_data])
179
- end
184
+ end
185
+ if opts[:user_data] && opts[:user_data].length > 0
186
+ params[:user_data]=Base64.encode64(opts[:user_data])
187
+ end
180
188
  safely do
181
189
  server = os.create_server(params)
182
- result = convert_from_server(server, os.connection.authuser)
190
+ result = convert_from_server(server, os.connection.authuser, get_attachments(server.id, os))
183
191
  end
184
192
  result
185
193
  end
@@ -189,7 +197,7 @@ module Deltacloud
189
197
  safely do
190
198
  server = os.get_server(instance_id)
191
199
  server.reboot! # sends a hard reboot (power cycle) - could instead server.reboot("SOFT")
192
- convert_from_server(server, os.connection.authuser)
200
+ convert_from_server(server, os.connection.authuser, get_attachments(instance_id, os))
193
201
  end
194
202
  end
195
203
 
@@ -222,7 +230,7 @@ module Deltacloud
222
230
  end
223
231
 
224
232
  def buckets(credentials, opts={})
225
- os = new_client(credentials, :buckets)
233
+ os = new_client(credentials, "object-store")
226
234
  buckets = []
227
235
  safely do
228
236
  if opts[:id]
@@ -235,7 +243,7 @@ module Deltacloud
235
243
  end
236
244
 
237
245
  def create_bucket(credentials, name, opts={})
238
- os = new_client(credentials, :buckets)
246
+ os = new_client(credentials, "object-store")
239
247
  bucket = nil
240
248
  safely do
241
249
  bucket = os.create_container(name)
@@ -244,14 +252,14 @@ module Deltacloud
244
252
  end
245
253
 
246
254
  def delete_bucket(credentials, name, opts={})
247
- os = new_client(credentials, :buckets)
255
+ os = new_client(credentials, "object-store")
248
256
  safely do
249
257
  os.delete_container(name)
250
258
  end
251
259
  end
252
260
 
253
261
  def blobs(credentials, opts={})
254
- os = new_client(credentials, :buckets)
262
+ os = new_client(credentials, "object-store")
255
263
  blobs = []
256
264
  safely do
257
265
  bucket = os.container(opts['bucket'])
@@ -265,7 +273,7 @@ module Deltacloud
265
273
  end
266
274
 
267
275
  def blob_data(credentials, bucket, blob, opts={})
268
- os = new_client(credentials, :buckets)
276
+ os = new_client(credentials, "object-store")
269
277
  safely do
270
278
  os.container(bucket).object(blob).data_stream do |chunk|
271
279
  yield chunk
@@ -274,30 +282,34 @@ module Deltacloud
274
282
  end
275
283
 
276
284
  def create_blob(credentials, bucket, blob, data, opts={})
277
- os = new_client(credentials, :buckets)
285
+ os = new_client(credentials, "object-store")
278
286
  safely do
279
- BlobHelper.rename_metadata_headers(opts, "X-Object-Meta-")
280
- os_blob = os.container(bucket).create_object(blob, {:content_type=> data[:type], :metadata=>opts}, data[:tempfile])
287
+ if(opts[:segment_manifest]) # finalize a segmented blob upload
288
+ os_blob = os.container(bucket).create_object(blob, {:manifest=>"#{bucket}/#{opts[:segmented_blob_id]}"})
289
+ else
290
+ BlobHelper.rename_metadata_headers(opts, "X-Object-Meta-")
291
+ os_blob = os.container(bucket).create_object(blob, {:content_type=> data[:type], :metadata=>opts}, data[:tempfile])
292
+ end
281
293
  convert_blob(os_blob, bucket)
282
294
  end
283
295
  end
284
296
 
285
297
  def delete_blob(credentials, bucket, blob, opts={})
286
- os = new_client(credentials, :buckets)
298
+ os = new_client(credentials, "object-store")
287
299
  safely do
288
300
  os.container(bucket).delete_object(blob)
289
301
  end
290
302
  end
291
303
 
292
304
  def blob_metadata(credentials, opts={})
293
- os = new_client(credentials, :buckets)
305
+ os = new_client(credentials, "object-store")
294
306
  safely do
295
307
  os.container(opts['bucket']).object(opts[:id]).metadata
296
308
  end
297
309
  end
298
310
 
299
311
  def update_blob_metadata(credentials, opts={})
300
- os = new_client(credentials, :buckets)
312
+ os = new_client(credentials, "object-store")
301
313
  safely do
302
314
  BlobHelper.rename_metadata_headers(opts["meta_hash"], "")
303
315
  blob = os.container(opts['bucket']).object(opts[:id])
@@ -305,8 +317,23 @@ module Deltacloud
305
317
  end
306
318
  end
307
319
 
320
+ def init_segmented_blob(credentials, opts={})
321
+ opts[:id]
322
+ end
323
+
324
+ def blob_segment_id(request, response)
325
+ #could be in http header OR query string:
326
+ segment_order = BlobHelper.segment_order(request)
327
+ blob_name = request.env["PATH_INFO"].gsub(/(&\w*=\w*)*$/, "").split("/").pop
328
+ "#{blob_name}#{segment_order}"
329
+ end
330
+
308
331
  #params: {:user,:password,:bucket,:blob,:content_type,:content_length,:metadata}
332
+ #params[:context] holds the request object - for getting to blob segment params
309
333
  def blob_stream_connection(params)
334
+ if BlobHelper.segmented_blob_op_type(params[:context]) == "segment"
335
+ params[:blob] = "#{params[:blob]}#{BlobHelper.segment_order(params[:context])}"
336
+ end
310
337
  tokens = params[:user].split("+")
311
338
  user_name, tenant_name = tokens.first, tokens.last
312
339
  #need a client for the auth_token and endpoints
@@ -352,10 +379,82 @@ module Deltacloud
352
379
  end
353
380
  end
354
381
 
382
+ def storage_volumes(credentials, opts={})
383
+ vs = new_client(credentials, "volume")
384
+ volumes = []
385
+ safely do
386
+ if opts[:id]
387
+ volumes << convert_volume(vs.get_volume(opts[:id]))
388
+ else
389
+ vs.volumes.each do |vol|
390
+ volumes << convert_volume(vol)
391
+ end
392
+ end
393
+ end
394
+ volumes
395
+ end
396
+
397
+ def create_storage_volume(credentials, opts=nil)
398
+ vs = new_client(credentials, "volume")
399
+ params = {}
400
+ safely do
401
+ params[:size] = opts.delete("capacity") || 1
402
+ params[:display_name] = opts.delete("name") || "Volume#{Time.now}"
403
+ params[:display_description] = opts.delete("description") || params[:display_name]
404
+ params[:availability_zone] = opts.delete("realm_id") unless (opts["realm_id"].nil? || opts["realm_id"].empty?)
405
+ opts.delete("commit")
406
+ opts.delete("snapshot_id") #FIXME AFTER ADDING SNAPSHOTS TO OPENSTACK GEM
407
+ volume = convert_volume(vs.create_volume(opts.merge(params)))
408
+ end
409
+ end
410
+
411
+ def destroy_storage_volume(credentials, opts={})
412
+ vs = new_client(credentials, "volume")
413
+ safely do
414
+ vs.delete_volume(opts[:id])
415
+ end
416
+ end
417
+
418
+ def attach_storage_volume(credentials, opts={})
419
+ vs = new_client(credentials, "volume")
420
+ cs = new_client(credentials, "compute")
421
+ safely do
422
+ cs.attach_volume(opts[:instance_id], opts[:id], opts[:device])
423
+ volume = convert_volume(vs.get_volume(opts[:id]))
424
+ end
425
+ end
426
+
427
+ def detach_storage_volume(credentials, opts={})
428
+ vs = new_client(credentials, "volume")
429
+ cs = new_client(credentials, "compute")
430
+ safely do
431
+ cs.detach_volume(opts[:instance_id], opts[:id])
432
+ volume = convert_volume(vs.get_volume(opts[:id]))
433
+ end
434
+ end
435
+
436
+ def storage_snapshots(credentials, opts={})
437
+ vs = new_client(credentials, "volume")
438
+ safely do
439
+ end
440
+ end
441
+
442
+ def create_storage_snapshot(credentials, opts={})
443
+ vs = new_client(credentials, "volume")
444
+ safely do
445
+ end
446
+ end
447
+
448
+ def destroy_storage_snapshot(credentials, opts={})
449
+ vs = new_client(credentials, "volume")
450
+ safely do
451
+ end
452
+ end
453
+
355
454
  private
356
455
 
357
456
  #for v2 authentication credentials.name == "username+tenant_name"
358
- def new_client(credentials, type = :compute)
457
+ def new_client(credentials, type = "compute")
359
458
  tokens = credentials.user.split("+")
360
459
  if credentials.user.empty?
361
460
  raise AuthenticationFailure.new(Exception.new("Error: you must supply the username"))
@@ -366,16 +465,10 @@ private
366
465
  user_name, tenant_name = tokens.first, tokens.last
367
466
  end
368
467
  safely do
369
- case type
370
- when :compute
371
- OpenStack::Connection.create(:username => user_name, :api_key => credentials.password, :authtenant => tenant_name, :auth_url => api_provider)
372
- when :buckets
373
- OpenStack::Connection.create(:username => user_name, :api_key => credentials.password, :authtenant => tenant_name, :auth_url => api_provider, :service_type => "object-store")
374
- else
375
- raise ValidationFailure.new(Exception.new("Error: tried to initialise Openstack connection using" +
376
- " an unknown service_type: #{type}"))
377
- end
378
- end
468
+ raise ValidationFailure.new(Exception.new("Error: tried to initialise Openstack connection using" +
469
+ " an unknown service_type: #{type}")) unless ["volume", "compute", "object-store"].include? type
470
+ OpenStack::Connection.create(:username => user_name, :api_key => credentials.password, :authtenant => tenant_name, :auth_url => api_provider, :service_type => type)
471
+ end
379
472
  end
380
473
 
381
474
  #NOTE: for the convert_from_foo methods below... openstack-compute
@@ -407,7 +500,7 @@ private
407
500
  })
408
501
  end
409
502
 
410
- def convert_from_server(server, owner)
503
+ def convert_from_server(server, owner, attachments=[])
411
504
  op = (server.class == Hash)? :fetch : :send
412
505
  image = server.send(op, :image)
413
506
  flavor = server.send(op, :flavor)
@@ -430,7 +523,8 @@ private
430
523
  :private_addresses => convert_server_addresses(server, :private),
431
524
  :username => 'root',
432
525
  :password => password,
433
- :keyname => server.send(op, :key_name)
526
+ :keyname => server.send(op, :key_name),
527
+ :storage_volumes => attachments.inject([]){|res, cur| res << {cur[:volumeId] => cur[:device]} ;res}
434
528
  )
435
529
  inst.actions = instance_actions_for(inst.state)
436
530
  inst.create_image = 'RUNNING'.eql?(inst.state)
@@ -447,10 +541,12 @@ private
447
541
  "STOPPED"
448
542
  when /build.*$/
449
543
  "PENDING"
544
+ when /error.*/
545
+ "ERROR"
450
546
  when /active/
451
547
  "RUNNING"
452
548
  else
453
- "UNKOWN"
549
+ "UNKNOWN"
454
550
  end
455
551
  end
456
552
 
@@ -489,6 +585,27 @@ private
489
585
  )
490
586
  end
491
587
 
588
+ def get_attachments(server_id, client)
589
+ if client.api_extensions[:"os-volumes"]
590
+ attachments = client.list_attachments(server_id)
591
+ attachments[:volumeAttachments] || []
592
+ else
593
+ []
594
+ end
595
+ end
596
+
597
+ def convert_volume(vol)
598
+ StorageVolume.new({ :id => vol.id,
599
+ :name => vol.display_name,
600
+ :created => vol.created_at,
601
+ :state => (vol.attachments.inject([]){|res, cur| res << cur if cur.size > 0 ; res}.empty?) ? "AVAILABLE" : "IN-USE",
602
+ :capacity => vol.size,
603
+ :instance_id => (vol.attachments.first["serverId"] unless vol.attachments.empty?),
604
+ :device => (vol.attachments.first["device"] unless vol.attachments.empty?),
605
+ :realm_id => vol.availability_zone,
606
+ :description => vol.display_description # openstack volumes have a display_description attr
607
+ })
608
+ end
492
609
 
493
610
  #IN: path1='server_path1'. content1='contents1', path2='server_path2', content2='contents2' etc
494
611
  #OUT:{local_path=>server_path, local_path1=>server_path2 etc}