fog-aws 3.0.0 → 3.10.0

data/Gemfile

@@ -11,4 +11,4 @@ end
 group :test do
   gem "simplecov"
   gem "codeclimate-test-reporter", "~> 1.0.0"
-end
+end

data/LICENSE.md

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014-2017 [CONTRIBUTORS.md](https://github.com/fog/fog-aws/blob/master/CONTRIBUTORS.md)
+Copyright (c) 2014-2019 [CONTRIBUTORS.md](https://github.com/fog/fog-aws/blob/master/CONTRIBUTORS.md)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

data/README.md

@@ -1,12 +1,10 @@
 # Fog::Aws
 
 ![Gem Version](https://badge.fury.io/rb/fog-aws.svg)
-[![Build Status](https://travis-ci.org/fog/fog-aws.svg?branch=master)](https://travis-ci.org/fog/fog-aws)
-[![Dependency Status](https://gemnasium.com/fog/fog-aws.svg)](https://gemnasium.com/fog/fog-aws)
+[![Build Status](https://github.com/fog/fog-aws/actions/workflows/ruby.yml/badge.svg)](https://github.com/fog/fog-aws/workflows/ruby.yml)
 [![Test Coverage](https://codeclimate.com/github/fog/fog-aws/badges/coverage.svg)](https://codeclimate.com/github/fog/fog-aws)
 [![Code Climate](https://codeclimate.com/github/fog/fog-aws.svg)](https://codeclimate.com/github/fog/fog-aws)
 
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -39,7 +37,10 @@ default:
   aws_secret_access_key: <YOUR_SECRET_ACCESS_KEY>
 ```
 
-### Connecting to EC2 service
+### EC2
+
+#### Connecting to the EC2 Service:
+
 ```ruby
 ec2 = Fog::Compute.new :provider => 'AWS', :region => 'us-west-2'
 ```
@@ -50,7 +51,7 @@ You can review all the requests available with this service using ```#requests``
 ec2.requests # => [:allocate_address, :assign_private_ip_addresses, :associate_address, ...]
 ```
 
-### Launch an EC2 on-demand instance:
+#### Launch an EC2 on-demand instance:
 
 ```ruby
 response = ec2.run_instances(
@@ -67,19 +68,71 @@ instance.wait_for { ready? }
 puts instance.public_ip_address # => "356.300.501.20"
 ```
 
-### Terminate an EC2 instance:
+#### Terminate an EC2 instance:
 
 ```ruby
 instance = ec2.servers.get("i-02db5af4")
 instance.destroy
 ```
 
-Fog::AWS is more than EC2 since it supports many services provided by AWS. The best way to learn and to know about how many services are supported is to take a look at the source code. To review the tests directory and to play with the library in ```irb``` can be very helpful resources as well.
+`Fog::AWS` is more than EC2 since it supports many services provided by AWS. The best way to learn and to know about how many services are supported is to take a look at the source code. To review the tests directory and to play with the library in ```bin/console``` can be very helpful resources as well.
+
+### S3
+
+#### Connecting to the S3 Service:
+
+```ruby
+s3 = Fog::Storage.new(provider: 'AWS', region: 'eu-central-1')
+```
+
+#### Creating a file:
+
+```ruby
+directory = s3.directories.new(key: 'gaudi-portal-dev')
+file = directory.files.create(key: 'user/1/Gemfile', body: File.open('Gemfile'), tags: 'Org-Id=1&Service-Name=My-Service')
+```
+
+#### Listing files:
+
+```ruby
+directory = s3.directories.get('gaudi-portal-dev', prefix: 'user/1/')
+directory.files
+```
+
+#### Generating a URL for a file:
+
+```ruby
+directory.files.new(key: 'user/1/Gemfile').url(Time.now + 60)
+```
+
+#### Copying a file
+
+```ruby
+directory = s3.directories.new(key: 'gaudi-portal-dev')
+file = directory.files.get('user/1/Gemfile')
+file.copy("target-bucket", "user/2/Gemfile.copy")
+```
+
+To speed transfers of large files, the `concurrency` option can be used
+to spawn multiple threads. Note that the file must be at least 5 MB for
+multipart uploads to work. For example:
+
+```ruby
+directory = s3.directories.new(key: 'gaudi-portal-dev')
+file = directory.files.get('user/1/Gemfile')
+file.multipart_chunk_size = 10 * 1024 * 1024
+file.concurrency = 10
+file.copy("target-bucket", "user/2/Gemfile.copy")
+```
 
 ## Documentation
 
 See the [online documentation](http://www.rubydoc.info/github/fog/fog-aws) for a complete API reference.
 
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
 ## Contributing
 
 1. Fork it ( https://github.com/fog/fog-aws/fork )

data/Rakefile

@@ -1,4 +1,5 @@
 require "bundler/gem_tasks"
+require "github_changelog_generator/task"
 
 task :default => :test
 
@@ -6,3 +7,8 @@ mock = ENV['FOG_MOCK'] || 'true'
 task :test do
   sh("export FOG_MOCK=#{mock} && bundle exec shindont")
 end
+
+GitHubChangelogGenerator::RakeTask.new :changelog do |config|
+  config.user = 'fog'
+  config.project = 'fog-aws'
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fog-aws"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/fog-aws.gemspec

@@ -15,15 +15,16 @@ Gem::Specification.new do |spec|
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
   spec.required_ruby_version = '>= 2.0.0'
 
-  spec.add_development_dependency 'bundler', '~> 1.15'
-  spec.add_development_dependency 'rake',    '~> 10.0'
-  spec.add_development_dependency 'rubyzip', '~> 1.2.1'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'github_changelog_generator', '~> 1.16'
+  spec.add_development_dependency 'rake',    '>= 12.3.3'
+  spec.add_development_dependency 'rubyzip', '~> 1.3.0'
   spec.add_development_dependency 'shindo',  '~> 0.3'
 
   spec.add_dependency 'fog-core',  '~> 2.1'

data/lib/fog/aws.rb

@@ -5,22 +5,6 @@ require 'fog/json'
 require File.expand_path('../aws/version', __FILE__)
 
 module Fog
-  module CDN
-    autoload :AWS,  File.expand_path('../aws/cdn', __FILE__)
-  end
-
-  module Compute
-    autoload :AWS, File.expand_path('../aws/compute', __FILE__)
-  end
-
-  module DNS
-    autoload :AWS, File.expand_path('../aws/dns', __FILE__)
-  end
-
-  module Storage
-    autoload :AWS, File.expand_path('../aws/storage', __FILE__)
-  end
-
   module AWS
     extend Fog::Provider
 
@@ -32,13 +16,17 @@ module Fog
 
     # Services
     autoload :AutoScaling,      File.expand_path('../aws/auto_scaling', __FILE__)
+    autoload :CDN,              File.expand_path('../aws/cdn', __FILE__)
     autoload :CloudFormation,   File.expand_path('../aws/cloud_formation', __FILE__)
     autoload :CloudWatch,       File.expand_path('../aws/cloud_watch', __FILE__)
+    autoload :Compute,          File.expand_path('../aws/compute', __FILE__)
     autoload :DataPipeline,     File.expand_path('../aws/data_pipeline', __FILE__)
+    autoload :DNS,              File.expand_path('../aws/dns', __FILE__)
     autoload :DynamoDB,         File.expand_path('../aws/dynamodb', __FILE__)
     autoload :ECS,              File.expand_path('../aws/ecs', __FILE__)
     autoload :EFS,              File.expand_path('../aws/efs', __FILE__)
     autoload :ELB,              File.expand_path('../aws/elb', __FILE__)
+    autoload :ELBV2,            File.expand_path('../aws/elbv2', __FILE__)
     autoload :EMR,              File.expand_path('../aws/emr', __FILE__)
     autoload :ElasticBeanstalk, File.expand_path('../aws/beanstalk', __FILE__)
     autoload :Elasticache,      File.expand_path('../aws/elasticache', __FILE__)
@@ -54,6 +42,7 @@ module Fog
     autoload :SNS,              File.expand_path('../aws/sns', __FILE__)
     autoload :SQS,              File.expand_path('../aws/sqs', __FILE__)
     autoload :STS,              File.expand_path('../aws/sts', __FILE__)
+    autoload :Storage,          File.expand_path('../aws/storage', __FILE__)
     autoload :Support,          File.expand_path('../aws/support', __FILE__)
     autoload :SimpleDB,         File.expand_path('../aws/simpledb', __FILE__)
 
@@ -70,6 +59,7 @@ module Fog
     service(:ecs,             'ECS')
     service(:efs,             'EFS')
     service(:elb,             'ELB')
+    service(:elbv2,           'ELBV2')
     service(:emr,             'EMR')
     service(:federation,      'Federation')
     service(:glacier,         'Glacier')
@@ -207,7 +197,7 @@ module Fog
         options = group_name
       elsif group_name
         if options.key?('GroupName')
-          raise Fog::Compute::AWS::Error, 'Arguments specified both group_name and GroupName in options'
+          raise Fog::AWS::Compute::Error, 'Arguments specified both group_name and GroupName in options'
         end
         options = options.clone
         options['GroupName'] = group_name
@@ -215,7 +205,7 @@ module Fog
       name_specified = options.key?('GroupName') && !options['GroupName'].nil?
       group_id_specified = options.key?('GroupId') && !options['GroupId'].nil?
       unless name_specified || group_id_specified
-        raise Fog::Compute::AWS::Error, 'Neither GroupName nor GroupId specified'
+        raise Fog::AWS::Compute::Error, 'Neither GroupName nor GroupId specified'
       end
       if name_specified && group_id_specified
         options.delete('GroupName')
@@ -229,7 +219,25 @@ module Fog
     end
 
     def self.regions
-      @regions ||= ['ap-northeast-1', 'ap-northeast-2', 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1', 'ca-central-1', 'eu-west-1', 'eu-west-2', 'eu-west-3', 'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'sa-east-1', 'cn-north-1', 'us-gov-west-1', 'ap-south-1']
+      @regions ||= [
+        'af-south-1',
+        'ap-east-1',
+        'ap-northeast-1', 'ap-northeast-2', 'ap-northeast-3',
+        'ap-south-1',
+        'ap-southeast-1', 'ap-southeast-2',
+        'ca-central-1',
+        'cn-north-1',
+        'cn-northwest-1',
+        'eu-central-1',
+        'eu-north-1',
+        'eu-west-1', 'eu-west-2', 'eu-west-3', 'eu-south-1',
+        'me-south-1',
+        'us-east-1', 'us-east-2',
+        'us-west-1', 'us-west-2',
+        'sa-east-1',
+        'us-gov-east-1',
+        'us-gov-west-1'
+      ]
     end
 
     def self.validate_region!(region, host=nil)

data/lib/fog/aws/auto_scaling.rb

@@ -48,6 +48,7 @@ module Fog
       request :resume_processes
       request :set_desired_capacity
       request :set_instance_health
+      request :set_instance_protection
       request :suspend_processes
       request :terminate_instance_in_auto_scaling_group
       request :update_auto_scaling_group

data/lib/fog/aws/cdn.rb

@@ -1,6 +1,6 @@
 module Fog
-  module CDN
-    class AWS < Fog::Service
+  module AWS
+    class CDN < Fog::Service
       extend Fog::AWS::CredentialFetcher::ServiceMethods
 
       requires :aws_access_key_id, :aws_secret_access_key
@@ -200,4 +200,17 @@ EOF
       end
     end
   end
+
+  # @deprecated
+  module CDN
+    # @deprecated
+    class AWS < Fog::AWS::CDN
+      # @deprecated
+      # @overrides Fog::Service.new (from the fog-core gem)
+      def self.new(*)
+        Fog::Logger.deprecation 'Fog::CDN::AWS is deprecated, please use Fog::AWS::CDN.'
+        super
+      end
+    end
+  end
 end

data/lib/fog/aws/compute.rb

@@ -1,12 +1,12 @@
 module Fog
-  module Compute
-    class AWS < Fog::Service
+  module AWS
+    class Compute < Fog::Service
       extend Fog::AWS::CredentialFetcher::ServiceMethods
 
       class RequestLimitExceeded < Fog::Errors::Error; end
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :endpoint, :region, :host, :path, :port, :scheme, :persistent, :aws_session_token, :use_iam_profile, :aws_credentials_expire_at, :instrumentor, :instrumentor_name, :version
+      recognizes :endpoint, :region, :host, :path, :port, :scheme, :persistent, :aws_session_token, :use_iam_profile, :aws_credentials_expire_at, :instrumentor, :instrumentor_name, :version, :retry_request_limit_exceeded, :retry_jitter_magnitude
 
       secrets    :aws_secret_access_key, :hmac, :aws_session_token
 
@@ -101,6 +101,7 @@ module Fog
       request :describe_classic_link_instances
       request :describe_dhcp_options
       request :describe_images
+      request :describe_image_attribute
       request :describe_instances
       request :describe_instance_attribute
       request :describe_internet_gateways
@@ -143,6 +144,7 @@ module Fog
       request :import_key_pair
       request :modify_image_attribute
       request :modify_instance_attribute
+      request :modify_instance_placement
       request :modify_network_interface_attribute
       request :modify_snapshot_attribute
       request :modify_subnet_attribute
@@ -452,7 +454,7 @@ module Fog
 
         def tagged_resources(resources)
           Array(resources).map do |resource_id|
-            if match = resource_id.match(/^(\w+)-[a-z0-9]{8}/i)
+            if match = resource_id.match(/^(\w+)-[a-z0-9]{8,17}/i)
               id = match.captures.first
             else
               raise(Fog::Service::NotFound.new("Unknown resource id #{resource_id}"))
@@ -546,11 +548,13 @@ module Fog
 
         def initialize(options={})
 
-          @connection_options     = options[:connection_options] || {}
-          @region                 = options[:region] ||= 'us-east-1'
-          @instrumentor           = options[:instrumentor]
-          @instrumentor_name      = options[:instrumentor_name] || 'fog.aws.compute'
-          @version                = options[:version]     ||  '2016-11-15'
+          @connection_options           = options[:connection_options] || {}
+          @region                       = options[:region] ||= 'us-east-1'
+          @instrumentor                 = options[:instrumentor]
+          @instrumentor_name            = options[:instrumentor_name] || 'fog.aws.compute'
+          @version                      = options[:version]     ||  '2016-11-15'
+          @retry_request_limit_exceeded = options.fetch(:retry_request_limit_exceeded, true)
+          @retry_jitter_magnitude       = options[:retry_jitter_magnitude] || 0.1
 
           @use_iam_profile = options[:use_iam_profile]
           setup_credentials(options)
@@ -632,30 +636,39 @@ module Fog
             raise if match.empty?
             raise case match[:code]
                 when 'NotFound', 'Unknown'
-                  Fog::Compute::AWS::NotFound.slurp(error, match[:message])
-                when 'RequestLimitExceeded'
-                  if retries < max_retries
-                    jitter = rand(100)
-                    waiting = true
-                    start_time = Time.now
-                    wait_time = ((2.0 ** (1.0 + retries) * 100) + jitter) / 1000.0
+                  Fog::AWS::Compute::NotFound.slurp(error, match[:message])
+                when 'RequestLimitExceeded'                  
+                  if @retry_request_limit_exceeded && retries < max_retries
+                    jitter = rand * 10 * @retry_jitter_magnitude
+                    wait_time = ((2.0 ** (1.0 + retries) * 100) / 1000.0) + jitter
                     Fog::Logger.warning "Waiting #{wait_time} seconds to retry."
-                    while waiting
-                      if Time.now - start_time >= wait_time
-                        waiting = false
-                      end
-                    end
+                    sleep(wait_time)
                     retries += 1
                     retry
+                  elsif @retry_request_limit_exceeded
+                    Fog::AWS::Compute::RequestLimitExceeded.slurp(error, "Max retries exceeded (#{max_retries}) #{match[:code]} => #{match[:message]}")
                   else
-                    Fog::Compute::AWS::RequestLimitExceeded.slurp(error, "Max retries exceeded (#{max_retries}) #{match[:code]} => #{match[:message]}")
+                    Fog::AWS::Compute::RequestLimitExceeded.slurp(error, "#{match[:code]} => #{match[:message]}")
                   end
                 else
-                  Fog::Compute::AWS::Error.slurp(error, "#{match[:code]} => #{match[:message]}")
+                  Fog::AWS::Compute::Error.slurp(error, "#{match[:code]} => #{match[:message]}")
                 end
           end
         end
       end
     end
   end
+
+  # @deprecated
+  module Compute
+    # @deprecated
+    class AWS < Fog::AWS::Compute
+      # @deprecated
+      # @overrides Fog::Service.new (from the fog-core gem)
+      def self.new(*)
+        Fog::Logger.deprecation 'Fog::Compute::AWS is deprecated, please use Fog::AWS::Compute.'
+        super
+      end
+    end
+  end
 end

data/lib/fog/aws/credential_fetcher.rb

@@ -1,17 +1,24 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
 module Fog
   module AWS
     module CredentialFetcher
 
       INSTANCE_METADATA_HOST = "http://169.254.169.254"
+      INSTANCE_METADATA_TOKEN = "/latest/api/token"
       INSTANCE_METADATA_PATH = "/latest/meta-data/iam/security-credentials/"
       INSTANCE_METADATA_AZ = "/latest/meta-data/placement/availability-zone/"
 
       CONTAINER_CREDENTIALS_HOST = "http://169.254.170.2"
 
+      STS_GLOBAL_ENDPOINT = "https://sts.amazonaws.com"
+
       module ServiceMethods
         def fetch_credentials(options)
           if options[:use_iam_profile] && Fog.mocking?
-            return Fog::Compute::AWS::Mock.data[:iam_role_based_creds]
+            return Fog::AWS::Compute::Mock.data[:iam_role_based_creds]
           end
           if options[:use_iam_profile]
             begin
@@ -22,25 +29,52 @@ module Fog
                 connection = options[:connection] || Excon.new(CONTAINER_CREDENTIALS_HOST)
                 credential_path = options[:credential_path] || ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
                 role_data = connection.get(:path => credential_path, :idempotent => true, :expects => 200).body
+                session = Fog::JSON.decode(role_data)
+
+                if region.nil?
+                  connection = options[:metadata_connection] || Excon.new(INSTANCE_METADATA_HOST)
+                  token_header = fetch_credentials_token_header(connection, options[:disable_imds_v2])
+                  region = connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200, :headers => token_header).body[0..-2]
+                end
+              elsif ENV["AWS_WEB_IDENTITY_TOKEN_FILE"]
+                params = {
+                  :Action => "AssumeRoleWithWebIdentity",
+                  :RoleArn => options[:role_arn] || ENV.fetch("AWS_ROLE_ARN"),
+                  :RoleSessionName => options[:role_session_name] || ENV["AWS_ROLE_SESSION_NAME"] || "fog-aws-#{SecureRandom.hex}",
+                  :WebIdentityToken => File.read(options[:aws_web_identity_token_file] || ENV.fetch("AWS_WEB_IDENTITY_TOKEN_FILE")),
+                  :Version => "2011-06-15",
+                }
+                connection = options[:connection] || Excon.new(STS_GLOBAL_ENDPOINT, :query => params)
+                document = Nokogiri::XML(connection.get(:idempotent => true, :expects => 200).body)
+
+                session = {
+                  "AccessKeyId" => document.css("AccessKeyId").children.text,
+                  "SecretAccessKey" => document.css("SecretAccessKey").children.text,
+                  "Token" => document.css("SessionToken").children.text,
+                  "Expiration" => document.css("Expiration").children.text,
+                }
 
                 if region.nil?
                   connection = options[:metadata_connection] || Excon.new(INSTANCE_METADATA_HOST)
-                  region = connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200).body[0..-2]
+                  token_header = fetch_credentials_token_header(connection, options[:disable_imds_v2])
+                  region = connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200, :headers => token_header).body[0..-2]
                 end
               else
                 connection = options[:connection] || Excon.new(INSTANCE_METADATA_HOST)
-                role_name = connection.get(:path => INSTANCE_METADATA_PATH, :idempotent => true, :expects => 200).body
-                role_data = connection.get(:path => INSTANCE_METADATA_PATH+role_name, :idempotent => true, :expects => 200).body
-                region ||= connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200).body[0..-2]
+                token_header = fetch_credentials_token_header(connection, options[:disable_imds_v2])
+                role_name = connection.get(:path => INSTANCE_METADATA_PATH, :idempotent => true, :expects => 200, :headers => token_header).body
+                role_data = connection.get(:path => INSTANCE_METADATA_PATH+role_name, :idempotent => true, :expects => 200, :headers => token_header).body
+                session = Fog::JSON.decode(role_data)
+                
+                region ||= connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200, :headers => token_header).body[0..-2]
               end
-
-              session = Fog::JSON.decode(role_data)
+              
               credentials = {}
               credentials[:aws_access_key_id] = session['AccessKeyId']
               credentials[:aws_secret_access_key] = session['SecretAccessKey']
               credentials[:aws_session_token] = session['Token']
               credentials[:aws_credentials_expire_at] = Time.xmlschema session['Expiration']
-
+              
               # set region by default to the one the instance is in.
               credentials[:region] = region
               #these indicate the metadata service is unavailable or has no profile setup
@@ -53,6 +87,26 @@ module Fog
             super
           end
         end
+
+        def fetch_credentials_token_header(connection, disable_imds_v2)
+          return nil if disable_imds_v2
+
+          token = connection.put(
+            :path => INSTANCE_METADATA_TOKEN,
+            :idempotent => true,
+            :expects => 200,
+            :retry_interval => 1,
+            :retry_limit => 3,
+            :read_timeout => 1,
+            :write_timeout => 1,
+            :connect_timeout => 1,
+            :headers => { "X-aws-ec2-metadata-token-ttl-seconds" => "300" }
+          ).body
+
+          { "X-aws-ec2-metadata-token" => token }
+        rescue Excon::Error
+          nil
+        end
       end
 
       module ConnectionMethods