vcloud-core 0.0.12 → 0.0.13

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.0.13 (2014-04-30)
+
+Feature:
+
+  - Remove support for Query API formats - we only ever use 'record' format.
+
 ## 0.0.12 (2014-04-22)
 
 Bugfix:

data/README.md

@@ -18,6 +18,44 @@ Or install it yourself as:
 
     $ gem install vcloud-core
 
+## Credentials
+
+vCloud Core is based around [fog](http://fog.io/). To use it you'll need to give it
+credentials that allow it to talk to a vCloud Director environment.
+
+1. Create a '.fog' file in your home directory.
+
+  For example:
+
+      test_credentials:
+        vcloud_director_host: 'host.api.example.com'
+        vcloud_director_username: 'username@org_name'
+        vcloud_director_password: ''
+
+2. Obtain a session token. First, curl the API:
+
+        curl -D- -d '' \
+            -H 'Accept: application/*+xml;version=5.1' -u '<username>@<org_name>' \
+            https://<host.api.example.com>/api/sessions
+
+  This will prompt for your password.
+
+  From the headers returned, the value of the `x-vcloud-authorization` header is your
+  session token, and this will be valid for 30 minutes idle - any activity will extend
+  its life by another 30 minutes.
+
+3. Specify your credentials and session token at the beginning of the command. For example:
+
+        FOG_CREDENTIAL=test_credentials \
+            FOG_VCLOUD_TOKEN=AAAABBBBBCCCCCCDDDDDDEEEEEEFFFFF= \
+            vcloud-query
+
+  You may find it easier to export one or both of the values as environment variables.
+
+  **NB** It is also possible to sidestep the need for the session token by saving your
+  password in the fog file. This is **not recommended**.
+
+
 ## VCloud Query
 
 ### Get results from the vCloud Query API
@@ -49,7 +87,7 @@ NB: examples assume FOG_CREDENTIAL or FOG_VCLOUD_TOKEN has been set accordingly.
     # Get general usage info
     vcloud-query --help
 
-    # Get a list of all queriable types (left column)
+    # Get a list of all queriable entity types
     vcloud-query
 
     # Get all VMs with VMware Tools less than 9282, that are not a vApp Template:
@@ -89,13 +127,29 @@ Entity metadata queries have their own subsyntax incorporating the value types:
 
 See http://pubs.vmware.com/vcd-51/topic/com.vmware.vcloud.api.doc_51/GUID-4FD71B6D-6797-4B8E-B9F0-618F4ACBEFAC.html for details.
 
+## The vCloud API
+
+vCloud Tools currently use version 5.1 of the [vCloud API](http://pubs.vmware.com/vcd-51/index.jsp?topic=%2Fcom.vmware.vcloud.api.doc_51%2FGUID-F4BF9D5D-EF66-4D36-A6EB-2086703F6E37.html). Version 5.5 may work but is not currently supported. You should be able to access the 5.1 API in a 5.5 environment, and this *is* currently supported.
+
+The default version is defined in [Fog](https://github.com/fog/fog/blob/244a049918604eadbcebd3a8eaaf433424fe4617/lib/fog/vcloud_director/compute.rb#L32).
+
+If you want to be sure you are pinning to 5.1, or use 5.5, you can set the API version to use in your fog file, e.g.
+
+`vcloud_director_api_version: 5.1`
+
+## Debugging
+
+`export EXCON_DEBUG=true` - this will print out the API requests and responses.
+
+`export DEBUG=true` - this will show you the stack trace when there is an exception instead of just the message.
+
 ## Testing
 
 Default target: `bundle exec rake`
 Runs the unit tests and feature tests.
 
 * Unit tests only: `bundle exec rake spec`
-* Feature tests only: `bundle exec rake feature`
+* Feature tests only: `bundle exec rake features`
 * Integration tests: `bundle exec rake integration`
 
 ### setting up and describing your environment for test runs

data/Rakefile

@@ -1,7 +1,7 @@
 require 'rspec/core/rake_task'
 require 'cucumber/rake/task'
 
-task :default => [:spec, :feature]
+task :default => [:spec, :features]
 
 RSpec::Core::RakeTask.new(:spec) do |task|
   # Set a bogus Fog credential, otherwise it's possible for the unit
@@ -12,7 +12,7 @@ RSpec::Core::RakeTask.new(:spec) do |task|
   task.pattern = FileList['spec/vcloud/**/*_spec.rb']
 end
 
-Cucumber::Rake::Task.new(:feature) do |t|
+Cucumber::Rake::Task.new(:features) do |t|
   t.cucumber_opts = "--format pretty --no-source"
   t.fork = false
 end

data/bin/vcloud-query

@@ -29,10 +29,6 @@ class App
     options[:fields] = v
   end
 
-  on('--format', '=ATTRIBUTE', 'Data format to retrieve: records, idrecords, references') do |v|
-    options[:format] = v
-  end
-
   on('--filter', '=FILTER', 'Filter expression') do |v|
     options[:filter] = v
   end

data/jenkins.sh

@@ -7,4 +7,5 @@ git clean -fdx
 bundle install --path "${HOME}/bundles/${JOB_NAME}"
 
 bundle exec rake
+RUBYOPT="-r ./tools/fog_credentials" bundle exec rake integration
 bundle exec rake publish_gem

data/jenkins_integration_tests.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -x
+set -e
+bundle install --path "${HOME}/bundles/${JOB_NAME}"
+RUBYOPT="-r ./tools/fog_credentials" bundle exec rake integration

data/lib/vcloud/core/query.rb

@@ -32,32 +32,11 @@ module Vcloud
     end
 
     def output_available_query_types
-      available_query_types = @query_runner.available_query_types
-      available_queries = collate_formats_for_types(available_query_types)
-      print_query_types(available_queries)
-    end
-
-    def collate_formats_for_types(available_queries)
-      queries = Hash.new { |h, k| h[k]=[] }
-      available_queries.each do |type, format|
-        queries[type] << format
-      end
-      queries
-    end
-
-    def print_query_types(queries)
-      type_width = longest_query_type(queries)
-
-      queries.keys.sort.each do |type|
-        puts "%-#{type_width}s %s" % [type, queries[type].sort.join(',')]
+      @query_runner.available_query_types.each do |entity_type|
+        puts entity_type
       end
     end
 
-    def longest_query_type(queries)
-      return 0 if queries.keys.empty?
-      queries.keys.max_by{|key| key.length}.length
-    end
-
     def output_header(results)
       return if results.size == 0
       case @options[:output_format]

data/lib/vcloud/core/query_runner.rb

@@ -6,21 +6,36 @@ module Vcloud
     end
 
     def run(type=nil, options={})
+      raise ArgumentError, "Query API :format option is not supported" if options[:format]
       get_all_results(type, options)
     end
 
     def available_query_types
-      query_list = @fsi.get_execute_query()
-      query_list[:Link].select do |link|
+      query_body = @fsi.get_execute_query()
+      get_entity_types_in_record_format(query_body)
+    end
+
+  private
+
+    def get_entity_types_in_record_format(query_body)
+      query_links = query_body.fetch(:Link).select do |link|
         link[:rel] == 'down'
-      end.map do |link|
-        href  = Nokogiri::XML.fragment(link[:href])
-        query = CGI.parse(URI.parse(href.text).query)
-        [query['type'].first, query['format'].first]
       end
+      entity_types = []
+      query_links.each do |link|
+        (entity_type, query_format) = extract_query_type_and_format_from_link(link)
+        entity_types << entity_type if query_format == 'records'
+      end
+      entity_types
     end
 
-  private
+    def extract_query_type_and_format_from_link(link)
+        href  = Nokogiri::XML.fragment(link[:href])
+        query = CGI.parse(URI.parse(href.text).query)
+        query_format = query['format'].first
+        query_type = query['type'].first
+        [query_type, query_format]
+    end
 
     def get_all_results(type, options)
       results = []

data/lib/vcloud/core/version.rb

@@ -1,5 +1,5 @@
 module Vcloud
   module Core
-    VERSION = '0.0.12'
+    VERSION = '0.0.13'
   end
 end

data/lib/vcloud/core/vm.rb

@@ -114,8 +114,7 @@ module Vcloud
 
       def generate_preamble(script_path, script_post_processor, vars)
         vapp_name = @vapp.name
-        script = ERB.new(File.read(File.expand_path(script_path)), nil, '>-')
-        .result(binding)
+        script = ERB.new(File.read(File.expand_path(script_path)), nil, '>-').result(binding)
         if script_post_processor
           script = Open3.capture2(File.expand_path(script_post_processor),
                                   stdin_data: script).first

data/spec/integration/edge_gateway/configure_edge_gateway_services_spec.rb

@@ -3,6 +3,20 @@ require 'spec_helper'
 module Vcloud
   module Core
     describe EdgeGateway do
+
+      required_env = {
+        'VCLOUD_EDGE_GATEWAY' => 'to name of VSE',
+      }
+
+      error = false
+      required_env.each do |var,message|
+        unless ENV[var]
+          puts "Must set #{var} #{message}" unless ENV[var]
+          error = true
+        end
+      end
+      Kernel.exit(2) if error
+
       it "configure firewall service" do
         configuration = {
             :FirewallService =>

data/spec/integration/edge_gateway/edge_gateway_spec.rb

@@ -3,6 +3,22 @@ require 'spec_helper'
 module Vcloud
   module Core
     describe EdgeGateway do
+
+      required_env = {
+        'VCLOUD_EDGE_GATEWAY' => 'to name of VSE',
+        'VCLOUD_PROVIDER_NETWORK_ID' => 'to ID of VSE external network',
+        'VCLOUD_NETWORK1_ID' => 'to the ID of a VSE internal network',
+      }
+
+      error = false
+      required_env.each do |var,message|
+        unless ENV[var]
+          puts "Must set #{var} #{message}" unless ENV[var]
+          error = true
+        end
+      end
+      Kernel.exit(2) if error
+
       context "get vcloud attributes for given gateway interface id " do
         before(:all) do
           @edge_gateway = EdgeGateway.get_by_name(ENV['VCLOUD_EDGE_GATEWAY'])

data/spec/integration/query/query_runner_spec.rb

@@ -0,0 +1,225 @@
+require 'spec_helper'
+
+module Vcloud
+  module Core
+    describe QueryRunner do
+
+      required_env = {
+        'VCLOUD_VDC_NAME' =>
+           'to the name of an orgVdc to use to instantiate vApps into',
+        'VCLOUD_TEMPLATE_NAME' =>
+           'to the name of a vAppTemplate to use create vApps in tests',
+        'VCLOUD_CATALOG_NAME' =>
+           'to the name of the catalog that VCLOUD_VAPP_TEMPLATE_NAME is stored in',
+      }
+
+      error = false
+      required_env.each do |var,message|
+        unless ENV[var]
+          puts "Must set #{var} #{message}" unless ENV[var]
+          error = true
+        end
+      end
+      Kernel.exit(2) if error
+
+      before(:all) do
+        @vapp_template_name = ENV['VCLOUD_TEMPLATE_NAME']
+        @vapp_template_catalog_name = ENV['VCLOUD_CATALOG_NAME']
+        @vdc_name = ENV['VCLOUD_VDC_NAME']
+      end
+
+      context "#available_query_types" do
+
+        before(:all) do
+          @query_types = QueryRunner.new.available_query_types
+        end
+
+        context "confirm accessing the query API is functional" do
+
+          it "returns an Array of available query types" do
+            expect(@query_types.class).to eq(Array)
+          end
+
+          it "returns at least one query type" do
+            expect(@query_types.size).to be >= 1
+          end
+
+        end
+
+        context "it supports all the vCloud entity types our tools need" do
+
+          it "supports the vApp entity type" do
+            expect(@query_types.include?("vApp")).to be_true
+          end
+
+          it "supports the vm entity type" do
+            expect(@query_types.include?("vm")).to be_true
+          end
+
+          it "supports the orgVdc entity type" do
+            expect(@query_types.include?("orgVdc")).to be_true
+          end
+
+          it "supports the orgVdcNetwork entity type" do
+            expect(@query_types.include?("orgVdcNetwork")).to be_true
+          end
+
+          it "supports the edgeGateway entity type" do
+            expect(@query_types.include?("edgeGateway")).to be_true
+          end
+
+          it "supports the task entity type" do
+            expect(@query_types.include?("task")).to be_true
+          end
+
+          it "supports the catalog entity type" do
+            expect(@query_types.include?("catalog")).to be_true
+          end
+
+          it "supports the catalogItem entity type" do
+            expect(@query_types.include?("catalogItem")).to be_true
+          end
+
+          it "supports the vAppTemplate entity type" do
+            expect(@query_types.include?("vAppTemplate")).to be_true
+          end
+
+        end
+
+      end
+
+      context "#run" do
+
+        before(:all) do
+          @number_of_vapps_to_create = 2
+          @test_case_vapps = create_test_case_vapps(
+            @number_of_vapps_to_create,
+            @vdc_name,
+            @vapp_template_catalog_name,
+            @vapp_template_name,
+          )
+        end
+
+        context "vApps are queriable with no options specified" do
+
+          before(:all) do
+            @all_vapps = QueryRunner.new.run('vApp')
+          end
+
+          it "returns an Array" do
+            expect(@all_vapps.class).to eq(Array)
+          end
+
+          it "returns at least the number of vApps that we created" do
+            expect(@all_vapps.size).to be >= @number_of_vapps_to_create
+          end
+
+          it "returns a record with a defined :name field" do
+            expect(@all_vapps.first[:name]).not_to be_empty
+          end
+
+          it "returns a record with a defined :href field" do
+            expect(@all_vapps.first[:href]).not_to be_empty
+          end
+
+          it "returns a record with a defined :vdcName field" do
+            expect(@all_vapps.first[:vdcName]).not_to be_empty
+          end
+
+          it "returns a record with a defined :status field" do
+            expect(@all_vapps.first[:status]).not_to be_empty
+          end
+
+          it "does not return a 'bogusElement' element" do
+            expect(@all_vapps.first.key?(:bogusElement)).to be false
+          end
+
+        end
+
+        context "Query output fields can be limited by supplying a comma-separated :fields list" do
+
+          before(:all) do
+            @results = QueryRunner.new.run('vApp', fields: "name,vdcName")
+          end
+
+          it "returns a record with a defined name element" do
+            expect(@results.first[:name]).not_to be_empty
+          end
+
+          it "returns a record with a defined vdcName element" do
+            expect(@results.first[:vdcName]).not_to be_empty
+          end
+
+          it "does not return a 'status' record, which we know is available for our vApp type" do
+            expect(@results.first.key?(:status)).to be false
+          end
+
+        end
+
+        context "Query API does not support an empty :fields list" do
+
+          it "raises a BadRequest exception, if empty string is supplied for :fields" do
+            expect { QueryRunner.new.run('vApp', fields: "") }.
+              to raise_exception(::Fog::Compute::VcloudDirector::BadRequest)
+          end
+
+        end
+
+        context "Query API returns href field regardless of filter :fields selected" do
+
+          it "returns href as well as name, if just 'name' is asked for" do
+            results = QueryRunner.new.run('vApp', fields: "name")
+            expect(results.first.keys.sort).to eq([:href, :name])
+          end
+
+          it "returns href, name, vdcName if 'name,vdcName' is asked for" do
+            results = QueryRunner.new.run('vApp', fields: "name,vdcName")
+            expect(results.first.keys.sort).to eq([:href, :name, :vdcName])
+          end
+
+        end
+
+        context "query output can be restricted by a filter expression on name" do
+
+          before(:all) do
+            @vapp_name = @test_case_vapps.last.name
+            @filtered_results = QueryRunner.new.run('vApp', filter: "name==#{@vapp_name}")
+          end
+
+          it "returns a single record matching our filter on name" do
+            expect(@filtered_results.size).to be(1)
+            expect(@filtered_results.first.fetch(:name)).to eq(@vapp_name)
+          end
+
+        end
+
+        after(:all) do
+          fsi = Vcloud::Fog::ServiceInterface.new()
+          @test_case_vapps.each do |vapp|
+            fsi.delete_vapp(vapp.id)
+          end
+        end
+
+        def create_test_case_vapps(quantity, vdc_name, catalog_name, vapp_template_name)
+          vapp_template = VappTemplate.get(catalog_name, vapp_template_name)
+          timestamp_in_s = Time.new.to_i
+          base_vapp_name = "vcloud-core-query-tests-#{timestamp_in_s}-"
+          network_names = []
+          vapp_list = []
+          quantity.times do |index|
+            vapp_list << Vapp.instantiate(
+              base_vapp_name + index.to_s,
+              network_names,
+              vapp_template.id,
+              vdc_name
+            )
+          end
+          vapp_list
+        end
+
+      end
+
+    end
+  end
+end
+

data/spec/vcloud/core/query_runner_spec.rb

@@ -8,44 +8,48 @@ describe Vcloud::QueryRunner do
   end
 
   context '#available_query_types' do
-    it 'should return empty array if no data' do
-      @mock_fog_interface.stub(:get_execute_query).and_return({:Link => {}})
 
+    it 'should return empty array if no query type links are returned from API' do
+      @mock_fog_interface.stub(:get_execute_query).and_return({:Link => {}})
       result = @query_runner.available_query_types
-
       result.size.should == 0
     end
 
-    it 'should parse the query types returned' do
+    it 'should parse the format=records query hrefs into a list of entity types' do
       @mock_fog_interface.stub(:get_execute_query).and_return(
         {:Link => [
           {:rel  => 'down',
-           :href => 'query?type=alice&format=references'},
+           :href => 'query?type=alice&format=records'},
+          {:rel  => 'down',
+           :href => 'query?type=bob&format=records'},
+          {:rel  => 'down',
+           :href => 'query?type=charlie&format=records'},
         ]})
-
-      result = @query_runner.available_query_types
-
-      result.size.should == 1
-      result[0][0].should == 'alice'
-      result[0][1].should == 'references'
+      expect(@query_runner.available_query_types).to eq(['alice', 'bob', 'charlie'])
     end
 
-    it 'should return the set of data' do
+    it 'should ignore query links with format=references and format=idrecords' do
       @mock_fog_interface.stub(:get_execute_query).and_return(
         {:Link => [
           {:rel  => 'down',
            :href => 'query?type=alice&format=references'},
           {:rel  => 'down',
-           :href => 'query?type=bob&format=references'},
+           :href => 'query?type=bob&format=idrecords'},
+          {:rel  => 'down',
+           :href => 'query?type=charlie&format=records'},
         ]})
-
-      result = @query_runner.available_query_types
-
-      result.size.should == 2
+      expect(@query_runner.available_query_types).to eq(['charlie'])
     end
+
   end
 
   context '#run' do
+
+    it "should raise an error if a :format option is supplied" do
+      expect { @query_runner.run('vApp', :format => 'references') }.
+        to raise_error(ArgumentError, "Query API :format option is not supported")
+    end
+
     it 'should return no results when fog returns no results' do
       @mock_fog_interface.stub(:get_execute_query).and_return({})
 

data/spec/vcloud/core/query_spec.rb

@@ -5,20 +5,14 @@ describe Vcloud::Query do
 
     context "#run called with no type set on construction" do
 
-      it "should get and reformat query types" do
+      it "should output all types that are available" do
         query_runner = double(Vcloud::QueryRunner)
-        allow(query_runner).to receive(:available_query_types) {
-          [
-            ['alice', 'references'],
-            ['alice', 'records'],
-            ['bob', 'records']
-          ]
-        }
+        allow(query_runner).to receive(:available_query_types) { [ 'alice', 'bob' ] }
 
         @query = Vcloud::Query.new(nil, {}, query_runner)
 
-        @query.should_receive(:puts).with("alice records,references")
-        @query.should_receive(:puts).with("bob   records")
+        @query.should_receive(:puts).with("alice")
+        @query.should_receive(:puts).with("bob")
 
         @query.run
       end

data/tools/fog_credentials.rb

@@ -0,0 +1,17 @@
+# Initialiser for getting vCloud credentials into Fog from Jenkins build
+# parameters, without needing to write them to disk. To be used with:
+#
+#     RUBYOPT="-r ./tools/fog_credentials" bundle exec integration
+#
+# Replace with FOG_VCLOUD_TOKEN support when we have a tool:
+#
+#     https://www.pivotaltracker.com/story/show/68989754
+#
+require 'bundler/setup'
+require 'fog'
+
+Fog.credentials = {
+  :vcloud_director_host     => ENV['API_HOST'],
+  :vcloud_director_username => ENV['API_USERNAME'],
+  :vcloud_director_password => ENV['API_PASSWORD'],
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vcloud-core
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-22 00:00:00.000000000 Z
+date: 2014-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -158,6 +158,7 @@ files:
 - features/support/env.rb
 - features/vcloud-query.feature
 - jenkins.sh
+- jenkins_integration_tests.sh
 - lib/vcloud/core.rb
 - lib/vcloud/core/compute_metadata.rb
 - lib/vcloud/core/config_loader.rb
@@ -181,6 +182,7 @@ files:
 - lib/vcloud/fog/service_interface.rb
 - spec/integration/edge_gateway/configure_edge_gateway_services_spec.rb
 - spec/integration/edge_gateway/edge_gateway_spec.rb
+- spec/integration/query/query_runner_spec.rb
 - spec/spec_helper.rb
 - spec/support/stub_fog_interface.rb
 - spec/vcloud/core/config_loader_spec.rb
@@ -202,6 +204,7 @@ files:
 - spec/vcloud/core/vm_spec.rb
 - spec/vcloud/fog/fog_model_interface_spec.rb
 - spec/vcloud/fog/service_interface_spec.rb
+- tools/fog_credentials.rb
 - vcloud-core.gemspec
 homepage: http://github.com/alphagov/vcloud-core
 licenses:
@@ -224,7 +227,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3087589811732903826
+      hash: 4282482764039593007
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -236,6 +239,7 @@ test_files:
 - features/vcloud-query.feature
 - spec/integration/edge_gateway/configure_edge_gateway_services_spec.rb
 - spec/integration/edge_gateway/edge_gateway_spec.rb
+- spec/integration/query/query_runner_spec.rb
 - spec/spec_helper.rb
 - spec/support/stub_fog_interface.rb
 - spec/vcloud/core/config_loader_spec.rb