fog-openstack 0.1.22 → 0.1.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4390c3d7b0437021392d05e1ca9caeed69dbe040
-  data.tar.gz: 67acad217dadd323bb3325f3c7bff3b69738e1e5
+  metadata.gz: 57deefcd73bd729138eb23ed6b27a7d5edebd986
+  data.tar.gz: 3f0e6066357fcac8d96ef1acace0ecc03336de19
 SHA512:
-  metadata.gz: 5590632eb25483cdd94d9e18f1f0a5f611dd9136f12ef506a0226a0a1ed51e76eb844e8673069cc8614e11cb46749827826261fb125209c9de5fd142a5efd91c
-  data.tar.gz: a7d3cfed1e7470a50d61a69a636ef7c198532b5dab31d701248ad80210bfd46f8f5acc90cd828618d5f02492b26ae23fb5ac47071a2c1721ad78760f43ca9c0f
+  metadata.gz: be0002aaf6e52c2578b3ec290e5cbe1dd7ef3fb30e1c5d58827e0de2c8fef58a99b812cf6e64c131f36a09c7b3d025d00731a277da19fa96f0e1f7080c67f8a5
+  data.tar.gz: 76d3fffed28f2442446a69c41a71eeba4eb1684183417fb605ee16a524425cc7a93dc4fc8c57d9da76901107b97514eaf9d1cba2eec2c80228fc948780a3c6a3

data/.travis.yml

@@ -1,5 +1,7 @@
 language: ruby
 sudo: false
+before_install:
+  - gem update bundler
 script:
   - bundle exec rake test
   - bundle exec rake spec
@@ -8,20 +10,14 @@ env:
 matrix:
   fast_finish: true
   include:
-  - rvm: 1.9.3
-    gemfile: gemfiles/Gemfile-1.9
-  - rvm: 2.0.0
-    gemfile: Gemfile
-  - rvm: 2.1.0
-    gemfile: Gemfile
-  - rvm: 2.1.1
-    gemfile: Gemfile
   - rvm: 2.2.0
     gemfile: Gemfile
   - rvm: 2.3.0
     gemfile: Gemfile
-  - rvm: jruby-19mode
-    gemfile: gemfiles/Gemfile-1.9
+  - rvm: 2.3.1
+    gemfile: Gemfile
+  - rvm: 2.4.2
+    gemfile: Gemfile
   - rvm: jruby-head
     gemfile: Gemfile
   allow_failures:

data/fog-openstack.gemspec

@@ -19,18 +19,18 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'fog-core',  '>= 1.40'
+  spec.required_ruby_version = '>= 2.1.0'
+
+  spec.add_dependency 'fog-core',  '~> 1.40'
   spec.add_dependency 'fog-json',  '>= 1.0'
   spec.add_dependency 'ipaddress', '>= 0.8'
 
   spec.add_development_dependency 'bundler', '~> 1.6'
-# TODO: remove when https://github.com/fog/fog-openstack/issues/202 is fixed
-#  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'coveralls'
   spec.add_development_dependency "mime-types"
   spec.add_development_dependency "mime-types-data"
   spec.add_development_dependency 'rake',    '~> 10.0'
   spec.add_development_dependency 'rubocop'
-  spec.add_development_dependency 'rubyzip', '~> 0.9.9'
   spec.add_development_dependency 'shindo',  '~> 0.3'
   spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'vcr'

data/lib/fog/compute/openstack.rb

@@ -1,8 +1,9 @@
-
-
 module Fog
   module Compute
     class OpenStack < Fog::Service
+      SUPPORTED_VERSIONS = /v2\.0|v2\.1/
+      SUPPORTED_MICROVERSION = '2.15'.freeze
+
       requires :openstack_auth_url
       recognizes :openstack_auth_token, :openstack_management_url,
                  :persistent, :openstack_service_type, :openstack_service_name,
@@ -380,6 +381,10 @@ module Fog
         end
 
         def initialize(options = {})
+          @supported_versions = SUPPORTED_VERSIONS
+          @supported_microversion = SUPPORTED_MICROVERSION
+          @microversion_key = 'X-OpenStack-Nova-API-Version'
+
           initialize_identity options
 
           @openstack_identity_service_type = options[:openstack_identity_service_type] || 'identity'
@@ -391,9 +396,9 @@ module Fog
 
           authenticate
 
-          unless @path =~ /1\.1|v2/
+          unless @path =~ %r{/(v2|v2\.0|v2\.1)/}
             raise Fog::OpenStack::Errors::ServiceUnavailable,
-                  "OpenStack compute binding only supports version 2 (a.k.a. 1.1)"
+                  "OpenStack compute binding only supports version v2 and v2.1"
           end
 
           @persistent = options[:persistent] || false

data/lib/fog/compute/openstack/models/server_group.rb

@@ -9,7 +9,7 @@ module Fog
         attribute :policies, :type => :array
         attribute :members
 
-        VALID_SERVER_GROUP_POLICIES = ['affinity', 'anti-affinity'].freeze
+        VALID_SERVER_GROUP_POLICIES = ['affinity', 'anti-affinity', 'soft-affinity', 'soft-anti-affinity'].freeze
 
         def self.validate_server_group_policy(policy)
           raise ArgumentError, "#{policy} is an invalid policy... must use one of #{VALID_SERVER_GROUP_POLICIES.join(', ')}" \

data/lib/fog/openstack.rb

@@ -81,6 +81,7 @@ module Fog
 
   module Orchestration
     autoload :OpenStack, File.expand_path('../orchestration/openstack', __FILE__)
+    autoload :Util, File.expand_path('../orchestration/util/recursive_hot_file_loader', __FILE__)
   end
 
   module SharedFileSystem

data/lib/fog/openstack/version.rb

@@ -1,11 +1,5 @@
 module Fog
   module Openstack
-    VERSION = "0.1.22"
-
-    def self.included(base)
-      if RUBY_VERSION < "2"
-        puts "DEPRECATION WARNING - Support for Ruby 1.9 will be dropped in fog-openstack 0.2 and higher. Please upgrade to Ruby 2 or above."
-      end
-    end
+    VERSION = "0.1.23"
   end
 end

data/lib/fog/orchestration/openstack/models/stack.rb

@@ -8,7 +8,7 @@ module Fog
 
         %w(capabilities description disable_rollback links notification_topics outputs parameters
            stack_name stack_status stack_status_reason template_description timeout_mins parent
-           creation_time updated_time stack_user_project_id stack_owner).each do |a|
+           creation_time updated_time stack_user_project_id stack_owner files).each do |a|
           attribute a.to_sym
         end
 
@@ -118,6 +118,7 @@ module Fog
           }
           options[:template] = template_content if template_content
           options[:template_url] = @template_url if @template_url
+          options[:files] = @files if @files
           options
         end
         private :default_options

data/lib/fog/orchestration/openstack/requests/create_stack.rb

@@ -10,6 +10,7 @@ module Fog
         #   * :template [String] Structure containing the template body.
         #   or (one of the two Template parameters is required)
         #   * :template_url [String] URL of file containing the template body.
+        #   * :files [Hash] Hash with files resources.
         #   * :disable_rollback [Boolean] Controls rollback on stack creation failure, defaults to false.
         #   * :parameters [Hash] Hash of providers to supply to template
         #   * :timeout_mins [Integer] Minutes to wait before status is set to CREATE_FAILED
@@ -28,6 +29,18 @@ module Fog
             }.merge(arg2.nil? ? {} : arg2)
           end
 
+          # Templates should always:
+          #  - be strings
+          #  - contain URI references instead of relative paths.
+          # Passing :template_url may not work well with `get_file` and remote `type`:
+          #  the python client implementation in shade retrieves from :template_uri
+          #  and replaces it with :template.
+          #  see https://github.com/openstack-infra/shade/blob/master/shade/openstackcloud.py#L1201
+          #  see https://developer.openstack.org/api-ref/orchestration/v1/index.html#create-stack
+          file_resolver = Util::RecursiveHotFileLoader.new(options[:template] || options[:template_url], options[:files])
+          options[:template] = file_resolver.template
+          options[:files] = file_resolver.files unless file_resolver.files.empty?
+
           request(
             :expects => 201,
             :path    => 'stacks',
@@ -68,6 +81,16 @@ module Fog
             'id'    => stack_id,
             'links' => [{"href" => "http://localhost:8004/v1/fake_tenant_id/stacks/#{options[:stack_name]}/#{stack_id}", "rel" => "self"}]
           }
+
+          if options.key?(:files)
+            response.body['files'] = {'foo.sh' => 'hello'}
+          end
+
+          if options.key?(:template) || options.key?(:template_url)
+            file_resolver = Util::RecursiveHotFileLoader.new(options[:template] || options[:template_url], options[:files])
+            response.body['files'] = file_resolver.files unless file_resolver.files.empty?
+          end
+
           response
         end
       end

data/lib/fog/orchestration/openstack/requests/update_stack.rb

@@ -10,6 +10,7 @@ module Fog
         #   or (one of the two Template parameters is required)
         #   * :template_url [String] URL of file containing the template body.
         #   * :parameters [Hash] Hash of providers to supply to template.
+        #   * :files [Hash] Hash with files resources.
         #
         def update_stack(arg1, arg2 = nil, arg3 = nil)
           if arg1.kind_of?(Stack)
@@ -17,22 +18,34 @@ module Fog
             stack = arg1
             stack_name = stack.stack_name
             stack_id = stack.id
-            params = arg2.nil? ? {} : arg2
+            options = arg2.nil? ? {} : arg2
           else
             # Deprecated, update_stack(stack_id, stack_name, options = {})
             Fog::Logger.deprecation("#update_stack(stack_id, stack_name, options) is deprecated, use #update_stack(stack, options) instead [light_black](#{caller.first})[/]")
             stack_id = arg1
             stack_name = arg2
-            params = {
+            options = {
               :stack_name => stack_name
             }.merge(arg3.nil? ? {} : arg3)
           end
 
+          # Templates should always:
+          #  - be strings
+          #  - contain URI references instead of relative paths.
+          # Passing :template_url may not work well with `get_file` and remote `type`:
+          #  the python client implementation in shade retrieves from :template_uri
+          #  and replaces it with :template.
+          #  see https://github.com/openstack-infra/shade/blob/master/shade/openstackcloud.py#L1201
+          #  see https://developer.openstack.org/api-ref/orchestration/v1/index.html#create-stack
+          file_resolver = Util::RecursiveHotFileLoader.new(options[:template] || options[:template_url], options[:files])
+          options[:template] = file_resolver.template
+          options[:files] = file_resolver.files unless file_resolver.files.empty?
+
           request(
             :expects => 202,
             :path    => "stacks/#{stack_name}/#{stack_id}",
             :method  => 'PUT',
-            :body    => Fog::JSON.encode(params)
+            :body    => Fog::JSON.encode(options)
           )
         end
       end
@@ -44,17 +57,26 @@ module Fog
             stack = arg1
             stack_name = stack.stack_name
             stack_id = stack.id
-            params = arg2.nil? ? {} : arg2
+            options = arg2.nil? ? {} : arg2
           else
             # Deprecated, update_stack(stack_id, stack_name, options = {})
             Fog::Logger.deprecation("#update_stack(stack_id, stack_name, options) is deprecated, use #update_stack(stack, options) instead [light_black](#{caller.first})[/]")
             stack_id = arg1
             stack_name = arg2
-            params = {
+            options = {
               :stack_name => stack_name
             }.merge(arg3.nil? ? {} : arg3)
           end
 
+          if options.key?(:files)
+            response.body['files'] = {'foo.sh' => 'hello'}
+          end
+
+          if options.key?(:template) || options.key?(:template_url)
+            file_resolver = Util::RecursiveHotFileLoader.new(options[:template] || options[:template_url], options[:files])
+            response.body['files'] = file_resolver.files unless file_resolver.files.empty?
+          end
+
           response = Excon::Response.new
           response.status = 202
           response.body = {}

data/lib/fog/orchestration/util/recursive_hot_file_loader.rb

@@ -0,0 +1,220 @@
+require 'set'
+require 'yaml'
+require 'open-uri'
+require 'objspace'
+require 'fog/core'
+
+module Fog
+  module Orchestration
+    module Util
+      #
+      # Resolve get_file resources found in a HOT template populating
+      #  a files Hash conforming to Heat Specs
+      #  https://developer.openstack.org/api-ref/orchestration/v1/index.html?expanded=create-stack-detail#stacks
+      #
+      # Files present in :files are not processed further. The others
+      #   are added to the Hash. This behavior is the same implemented in openstack-infra/shade
+      #   see https://github.com/openstack-infra/shade/blob/1d16f64fbf376a956cafed1b3edd8e51ccc16f2c/shade/openstackcloud.py#L1200
+      #
+      # This implementation just process nested templates but not resource
+      #  registries.
+      class RecursiveHotFileLoader
+        attr_reader :files
+        attr_reader :template
+
+        def initialize(template, files = nil)
+          # According to https://github.com/fog/fog-openstack/blame/master/docs/orchestration.md#L122
+          #  templates can be either String or Hash.
+          #  If it's an Hash, we deep_copy it so the passed argument
+          #  is not modified by get_file_contents.
+          template = deep_copy(template)
+          @visited = Set.new
+          @files = files || {}
+          @template = get_template_contents(template)
+        end
+
+        # Return string
+        def url_join(prefix, suffix)
+          if prefix
+            # URI.join replaces prefix parts before a
+            #  trailing slash. See https://docs.ruby-lang.org/en/2.3.0/URI.html.
+            prefix += '/' unless prefix.to_s.end_with?("/")
+            suffix = URI.join(prefix, suffix)
+            # Force URI to use traditional file scheme representation.
+            suffix.host = "" if suffix.scheme == "file"
+          end
+          suffix.to_s
+        end
+
+        # Retrieve a template content.
+        #
+        # @param template_file can be either:
+        #          - a raw_template string
+        #          - an URI string
+        #          - an Hash containing the parsed template.
+        #
+        # XXX: we could use named parameters
+        # and better mimic heatclient implementation.
+        def get_template_contents(template_file)
+          Fog::Logger.debug("get_template_contents [#{template_file}]")
+
+          raise "template_file should be Hash or String, not #{template_file.class.name}" unless
+            template_file.kind_of?(String) || template_file.kind_of?(Hash)
+
+          local_base_url = url_join("file:/", File.absolute_path(Dir.pwd))
+
+          if template_file.kind_of?(Hash)
+            template_base_url = local_base_url
+            template = template_file
+          elsif template_is_raw?(template_file)
+            template_base_url = local_base_url
+            template = YAML.safe_load(template_file, [Date])
+          elsif template_is_url?(template_file)
+            template_file = normalise_file_path_to_url(template_file)
+            template_base_url = base_url_for_url(template_file)
+            raw_template = read_uri(template_file)
+            template = YAML.safe_load(raw_template, [Date])
+
+            Fog::Logger.debug("Template visited: #{@visited}")
+            @visited.add(template_file)
+          else
+            raise "template_file is not a string of the expected form"
+          end
+
+          get_file_contents(template, template_base_url)
+
+          template
+        end
+
+        # Traverse the template tree looking for get_file and type
+        #   and populating the @files attribute with their content.
+        #   Resource referenced by get_file and type are eventually
+        #   replaced with their absolute URI as done in heatclient
+        #   and shade.
+        #
+        def get_file_contents(from_data, base_url)
+          Fog::Logger.debug("Processing #{from_data} with base_url #{base_url}")
+
+          # Recursively traverse the tree
+          #   if recurse_data is Array or Hash
+          recurse_data = from_data.kind_of?(Hash) ? from_data.values : from_data
+          recurse_data.each { |value| get_file_contents(value, base_url) } if recurse_data.kind_of?(Array)
+
+          # I'm on a Hash, process it.
+          return unless from_data.kind_of?(Hash)
+          from_data.each do |key, value|
+            next if ignore_if(key, value)
+
+            # Resolve relative paths.
+            str_url = url_join(base_url, value)
+
+            next if @files.key?(str_url)
+
+            file_content = read_uri(str_url)
+
+            # get_file should not recurse hot templates.
+            if key == "type" && template_is_raw?(file_content) && !@visited.include?(str_url)
+              template = get_template_contents(str_url)
+              file_content = YAML.dump(template)
+            end
+
+            @files[str_url] = file_content
+            # replace the data value with the normalised absolute URL as required
+            #  by https://docs.openstack.org/heat/pike/template_guide/hot_spec.html#get-file
+            from_data[key] = str_url
+          end
+        end
+
+        private
+
+        # Retrive the content of a local or remote file.
+        #
+        # @param A local or remote uri.
+        #
+        # @raise ArgumentError if it's not a valid uri
+        #
+        # Protect open-uri from malign arguments like
+        #  - "|ls"
+        #  - multiline strings
+        def read_uri(uri_or_filename)
+          remote_schemes = %w[http https ftp]
+          Fog::Logger.debug("Opening #{uri_or_filename}")
+
+          begin
+            # Validate URI to protect from open-uri attacks.
+            url = URI(uri_or_filename)
+
+            if remote_schemes.include?(url.scheme)
+              # Remote schemes must contain an host.
+              raise ArgumentError if url.host.nil?
+
+              # Encode URI with spaces.
+              uri_or_filename = uri_or_filename.gsub(/ /, "%20")
+            end
+          rescue URI::InvalidURIError
+            raise ArgumentError, "Not a valid URI: #{uri_or_filename}"
+          end
+
+          # TODO: A future revision may implement a retry.
+          content = ''
+          # open-uri doesn't open "file:///" uris.
+          uri_or_filename = uri_or_filename.sub(/^file:/, "")
+
+          open(uri_or_filename) { |f| content = f.read }
+          content
+        end
+
+        # Return true if the file is an heat template, false otherwise.
+        def template_is_raw?(content)
+          htv = content.strip.index("heat_template_version")
+          # Tolerate some leading character in case of a json template.
+          htv && htv < 5
+        end
+
+        # Return true if it's an URI, false otherwise.
+        def template_is_url?(path)
+          normalise_file_path_to_url(path)
+          true
+        rescue ArgumentError, URI::InvalidURIError
+          false
+        end
+
+        # Return true if I should I process this this file.
+        #
+        # @param [String] An heat template key
+        #
+        def ignore_if(key, value)
+          return true if key != 'get_file' && key != 'type'
+
+          return true unless value.kind_of?(String)
+
+          return true if key == 'type' &&
+                         !value.end_with?('.yaml', '.template')
+
+          false
+        end
+
+        # Returns the string baseurl of the given url.
+        def base_url_for_url(url)
+          parsed = URI(url)
+          parsed_dir = File.dirname(parsed.path)
+          url_join(parsed, parsed_dir)
+        end
+
+        def normalise_file_path_to_url(path)
+          # Nothing to do on URIs
+          return path if URI(path).scheme
+
+          path = File.absolute_path(path)
+          url_join('file:/', path)
+        end
+
+        def deep_copy(item)
+          return item if item.kind_of?(String)
+
+          YAML.safe_load(YAML.dump(item), [Date])
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: fog-openstack
 version: !ruby/object:Gem::Version
-  version: 0.1.22
+  version: 0.1.23
 platform: ruby
 authors:
 - Matt Darby
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-10-12 00:00:00.000000000 Z
+date: 2018-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.40'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.40'
 - !ruby/object:Gem::Dependency
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mime-types
   requirement: !ruby/object:Gem::Requirement
@@ -122,20 +136,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubyzip
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.9.9
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.9.9
 - !ruby/object:Gem::Dependency
   name: shindo
   requirement: !ruby/object:Gem::Requirement
@@ -245,7 +245,6 @@ files:
 - examples/volume/backups.rb
 - examples/workflow/workflow-examples.rb
 - fog-openstack.gemspec
-- gemfiles/Gemfile-1.9
 - lib/fog/baremetal/openstack.rb
 - lib/fog/baremetal/openstack/models/chassis.rb
 - lib/fog/baremetal/openstack/models/chassis_collection.rb
@@ -1043,6 +1042,7 @@ files:
 - lib/fog/orchestration/openstack/requests/show_stack_details.rb
 - lib/fog/orchestration/openstack/requests/update_stack.rb
 - lib/fog/orchestration/openstack/requests/validate_template.rb
+- lib/fog/orchestration/util/recursive_hot_file_loader.rb
 - lib/fog/planning/openstack.rb
 - lib/fog/planning/openstack/models/plan.rb
 - lib/fog/planning/openstack/models/plans.rb
@@ -1352,7 +1352,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/gemfiles/Gemfile-1.9

@@ -1,22 +0,0 @@
-source "https://rubygems.org"
-
-group :development, :test do
-  gem "fog-core",        ">= 1.35"
-  gem "fog-json",        ">= 1.0"
-  gem "fog-xml",         ">= 0.1"
-  gem "ipaddress",       ">= 0.8"
-  gem "bundler",         "~> 1.6"
-  gem "mime-types",      "2.6.2"
-  gem "nokogiri",        "< 1.7.0"
-  gem "rake",            "~> 10.0"
-  gem "rubocop",         "< 0.42"
-  gem "rubyzip",         "~> 0.9.9"
-  gem "shindo",          "~> 0.3"
-  gem "minitest"
-  gem "vcr"
-  gem "webmock",         "~> 1.24.6"
-  gem "json",            "< 2.0"
-# TODO: remove when https://github.com/fog/fog-openstack/issues/202 is fixed
-#  gem "coveralls",       "< 0.8.15"
-  gem "public_suffix",    "~> 1.4.6"
-end