chef-provisioning-aws 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 152994be990de9eda227b6a5be1179acea932c78
-  data.tar.gz: 1c58f8b2be7da9e3a779dae16ba2ff445de4370d
+  metadata.gz: 19f4ffce05e2b12c1256593ed8132affc94376f8
+  data.tar.gz: e606311b2689000c0e5098087d5debaff8c2997d
 SHA512:
-  metadata.gz: aba82b6a79304d5786b8b686aad781c18add3fac6b1d52c52b300dffd38ec93b2262cda6be721de582f5382cea04a08435f3a3001bd4857415743713a0ca4e66
-  data.tar.gz: 2e70c7a272b3902b7b8a96a70f4a70a99f9a6ee3d38e1ef3c329efef09f055129d2a97202d00dcc08f6ec0e3d32e171f908dea6eef093de97c853cb37ac9e9f3
+  metadata.gz: a37d0a594d564b0a6da4f84b8a4d9d6222654e7bc2ddedb68d22d830cdfea1321cfb4b3d024264e14c81a7462b1f47b3c311ebff79972deb8bba2b4558847f57
+  data.tar.gz: 8dd82f47ac3c83b52b124d75a316c1c54a348d755c2e892213cd245fdf5f5cfeabebfb67bc81eed26c9362d62ba6c4dbfa3f07a7ecf40dfcc7d00668d00aa800

data/README.md

@@ -10,6 +10,16 @@ AWS credentials should be specified in your `~/.aws/credentials` file as documen
 
 You can specify a profile as the middle section of the semi-colon seperated driver url.  For example, a driver url of `aws:staging:us-east-1` would use the profile `staging`.
 
+## Configurable Options
+
+When using `machine_batch` with a large number of machines it is possible to overwhelm the AWS SDK until it starts returning `AWS::EC2::Errors::RequestLimitExceeded`.  You can configure the AWS SDK to retry these errors automatically by specifying
+
+```ruby
+chef_provisioning({:aws_retry_limit => 10})
+```
+
+in your client.rb for the provisioning workstation.  The default `:aws_retry_limit` is 5.
+
 # Resources
 
 TODO: List out weird/unique things about resources here.  We don't need to document every resource
@@ -26,7 +36,7 @@ You can specify an existing key pair to upload by specifying the following:
 aws_key_pair 'my-aws-key' do
   private_key_path "~boiardi/.ssh/my-aws-key.pem"
   public_key_path "~boiardi/.ssh/my-aws-key.pub"
-  overwrite false # Set to true if you want to regenerate this each chef run
+  allow_overwrite false # Set to true if you want to regenerate this each chef run
 end
 ```
 
@@ -108,6 +118,7 @@ configure the machine.  These are all the available options:
 
 ```ruby
 with_machine_options({
+  # See https://github.com/chef/chef-provisioning#machine-options for options shared between drivers
   bootstrap_options: {
     # http://docs.aws.amazon.com/sdkforruby/api/Aws/EC2/Resource.html#create_instances-instance_method
     # lists the available options.  The below options are the default
@@ -117,45 +128,8 @@ with_machine_options({
     key_path: "~/.chef/keys/chef_default", # only necessary if storing keys some other location
     user_data: "...", # Only defaulted on Windows instances to start winrm
   },
-  convergence_options: {
-    chef_version: "12.4.1",
-    prerelease: "false",
-    chef_client_timeout: 120*60, # Default: 2 hours
-    chef_config: "log_level :debug\\n", # String containing additional text to inject into client.rb
-    chef_server: "http://my.chef.server/", # TODO could conflict with https://github.com/chef/chef-provisioning#pointing-boxes-at-chef-servers
-    bootstrap_proxy: "http://localhost:1234",
-    ssl_verify_mode: :verify_peer,
-    client_rb_path: "/etc/chef/client.rb", # <- DEFAULT, overwrite if necessary
-    client_pem_path: "/etc/chef/client.pem", # <- DEFAULT, overwrite if necessary
-    allow_overwrite_keys: false, # If there is an existing client.pem this needs to be true to overwrite it
-    private_key_options: {}, # TODO ????? Something to do with creating node object
-    source_key: "", # ?????
-    source_key_pass_phrase: "", # ?????
-    source_key_path: "", # ?????
-    public_key_path: "", # ?????
-    public_key_format: "", # ?????
-    admin: "", # ?????
-    validator: "", # ?????
-    ohai_hints: { :ec2 => { :key => :value } }, # Map from hint file name to file contents, this would create /etc/chef/ohai/hints/ec2.json
-    # The following are only available for Linux machines
-    install_sh_url: "https://www.chef.io/chef/install.sh", # <- DEFAULT, overwrite if necessary
-    install_sh_path: "/tmp/chef-install.sh", # <- DEFAULT, overwrite if necessary
-    install_sh_arguments: "-P chef-dk", # Additional commands to pass to install.sh
-    # The following are only available for Windows machines
-    install_msi_url: "foo://bar.com"
-  },
-  ssh_options: {
-    ...
-  },
-  cached_installer: false, # ???
-  aws_tags: { :key1 => "value", "key2" => "value"},
-  source_dest_check: false, # Specifies whether to enable an instance launched in a VPC to perform NAT
-  is_windows: false, # set to true if using a Windows AMI
-  ssh_username: "ubuntu",
-  ssh_gateway: "localhost"
-  sudo: true,
-  use_private_ip_for_ssh: false, # If set to true, we will access the instance with its private_ip (usually requires VPN access)
-  ...
+  use_private_ip_for_ssh: false, # DEPRECATED, use `transport_address_location`
+  transport_address_location: :public_ip, # `:public_ip` (default), `:private_ip` or `:dns`.  Defines how SSH or WinRM should find an address to communicate with the instance.
 })
 ```
 
@@ -192,24 +166,59 @@ load_balancer "my_elb" do
   })
 ```
 
-The available parameters for `load_balancer_options` can be viewed at http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/ELB/Client.html#create_load_balancer-instance_method .
+The available parameters for `load_balancer_options` can be viewed in the [aws docs](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/ELB/Client.html#create_load_balancer-instance_method).
 
 NOTES:
 
 1. You can specify either `ssl_certificate_id` or `server_certificate` in a listener but the value to both parameters should be the ARN of an existing IAM::ServerCertificate object.
 2. Instead of specifying `tags` in the `load_balancer_options`, you should specify `aws_tags`.  See the note on [tagging base resources](https://github.com/chef/chef-provisioning-aws#base-resources).
 
+# RDS Instance Options
+
+### Additional Options
+
+RDS instances have many options. Some of them live as first class attributes. Any valid RDS option that is not a first class attribute can still be set via a hash in `additional_options`.
+*If you set an attribute and also specify it in `additional_options`, the resource will chose the attribute and not what is specified in `additional_options`.*
+
+To illustrate, note that the following example defines `multi_az` as both an attribute and in the `additional_options` hash:
+
+```
+aws_rds_instance "test-rds-instance2" do
+  engine "postgres"
+  publicly_accessible false
+  db_instance_class "db.t1.micro"
+  master_username "thechief"
+  master_user_password "securesecure"
+  multi_az false
+  additional_options(multi_az: true)
+end
+```
+
+The above would result in a new `aws_rds_instance` with `multi_az` being `false`.
+
+Additional values for `additional_options` can view viewed in the [aws docs](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/RDS/Client.html#create_db_instance-instance_method).
+
+### Specifying a DB Subnet Group for your RDS Instance
+
+See [this example](docs/examples/aws_rds_subnet_group.rb) for how to set up a DB Subnet Group and pass it to your RDS Instance.
+
 # Specifying a Chef Server
 
 See [Pointing Boxes at Chef Servers](https://github.com/chef/chef-provisioning/blob/master/README.md#pointing-boxes-at-chef-servers)
 
 # Tagging Resources
 
-## Aws Resources
+## For Recipe authors
+
+All resources (incuding base resources like `machine`) that are taggable support an `aws_tags` attribute which accepts a single layer hash.  To set just the key of an AWS tag specify the value as nil.  EG, `aws_tags {my_tag_key: nil}`.  Some AWS objects cannot accept nil values and will automatically convert it to an empty string.
+
+Some AWS objects (may EC2) view the `Name` tag as unique - it shows up in a `Name` column in the AWS console.  By default we specify the `Name` tag as the resource name.  This can be overridden by specifying `aws_tags {Name: 'some other name'}`.
+
+You can remove all the tags _except_ the `Name` tag by specifying `aws_tags {}`.
 
-All resources which extend Chef::Provisioning::AWSDriver::AWSResourceWithEntry support the ability
-to add tags, except AwsEipAddress.  AWS does not support tagging on AwsEipAddress.  To add a tag
-to any aws resource, us the `aws_tags` attribute and provide it a hash:
+Tag keys and values can be specified as symbols or strings but will be converted to strings before sending to AWS.
+
+Examples:
 
 ```ruby
 aws_ebs_volume 'ref-volume' do
@@ -221,39 +230,31 @@ aws_vpc 'ref-vpc' do
 end
 ```
 
-The hash of tags can use symbols or strings for both keys and values.  The tags will be converged
-idempotently, meaning no write will occur if no tags are changing.
-
-We will not touch the `'Name'` tag UNLESS you specifically pass it.  If you do not pass it, we
-leave it alone.
+## For Resource Authors
 
-## Base Resources
+To enable tagging support you must make specific changes to the Resource and Attribute.  For the Resource it needs to include the `attribute aws_tags`.  This should be done by `include Chef::Provisioning::AWSDriver::AWSTaggable` on the Resource.
 
-Because base resources from chef-provisioning do not have the `aws_tag` attribute, they must be
-tagged in their options:
+The `AWSProvider` class will automatically try to call `converge_tags` when running the `action_create` method.  You should instantiate an instance of the `AWSTagger` and provide it a strategy depending on the client used to perform the tagging.  For example, an RDS Provider should define
 
 ```ruby
-machine 'ref-machine-1' do
-  machine_options :aws_tags => {:marco => 'polo', :happyhappy => 'joyjoy'}
-end
-
-machine_batch "ref-batch" do
-  machine 'ref-machine-2' do
-    machine_options :aws_tags => {:marco => 'polo', :happyhappy => 'joyjoy'}
-    converge false
-  end
-  machine 'ref-machine-3' do
-    machine_options :aws_tags => {:othercustomtags => 'byebye'}
-    converge false
+def aws_tagger
+  @aws_tagger ||= begin
+    rds_strategy = Chef::Provisioning::AWSDriver::TaggingStrategy::RDS.new(
+      new_resource.driver.rds.client,
+      construct_arn(new_resource),
+      new_resource.aws_tags
+    )
+    Chef::Provisioning::AWSDriver::AWSTagger.new(rds_strategy, action_handler)
   end
 end
-
-load_balancer 'ref-elb' do
-  load_balancer_options :aws_tags => {:marco => 'polo', :happyhappy => 'joyjoy'}
+def converge_tags
+  aws_tagger.converge_tags
 end
 ```
 
-See `docs/examples/aws_tags.rb` for further examples.
+The `aws_tagger` method is used by the tests to assert that the object tags are correct.  These methods can be encapsulated in an module for DRY purposes, as the EC2 strategy shows.
+
+Finally, you should add 3 standard tests for taggable objects - 1) Tags can be created on a new object, 2) Tags can be updated on an existing object with tags and 3) Tags can be cleared by setting `aws_tags {}`.  Copy the tests from an existing spec file and modify them to support your resource.  TODO make a module that copies these tests for us.  Right now it is complicated by the fact that some resources have required attributes that others don't.
 
 # Looking up AWS objects
 

data/Rakefile

@@ -4,14 +4,34 @@ require 'rspec/core/rake_task'
 
 task :default => :spec
 
-desc "Run specs"
+ENV['AWS_TEST_DRIVER'] ||= "aws"
+
+desc "run all non-integration specs"
 RSpec::Core::RakeTask.new(:spec) do |spec|
   spec.pattern = 'spec/**/*_spec.rb'
   # TODO add back integration tests whenever we have strategy for keys
   spec.exclude_pattern = 'spec/integration/**/*_spec.rb'
 end
 
-desc "Run Integration Specs"
+desc "run integration specs"
 RSpec::Core::RakeTask.new(:integration) do |spec|
   spec.pattern = 'spec/integration/**/*_spec.rb'
 end
+
+desc "run :super_slow specs (machine/machine_image)"
+RSpec::Core::RakeTask.new(:slow) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rspec_opts = "-t super_slow"
+end
+
+desc "run all specs, except :super_slow"
+RSpec::Core::RakeTask.new(:all) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+end
+
+desc "run all specs, including :super_slow"
+task :all_slow do
+  %w(all slow).each do |t|
+    Rake::Task[t].invoke
+  end
+end

data/lib/chef/provider/aws_auto_scaling_group.rb

@@ -2,11 +2,12 @@ require 'chef/provisioning/aws_driver/aws_provider'
 require 'set'
 
 class Chef::Provider::AwsAutoScalingGroup < Chef::Provisioning::AWSDriver::AWSProvider
+  provides :aws_auto_scaling_group
 
   protected
 
   def create_aws_object
-    converge_by "create new Auto Scaling Group #{new_resource.name} in #{region}" do
+    converge_by "create Auto Scaling group #{new_resource.name} in #{region}" do
       options = desired_options.dup
       options[:min_size] ||= 1
       options[:max_size] ||= 1
@@ -20,7 +21,7 @@ class Chef::Provider::AwsAutoScalingGroup < Chef::Provisioning::AWSDriver::AWSPr
   end
 
   def destroy_aws_object(group)
-    converge_by "delete Auto Scaling Group #{new_resource.name} in #{region}" do
+    converge_by "delete Auto Scaling group #{new_resource.name} in #{region}" do
       group.delete!
     end
   end

data/lib/chef/provider/aws_cache_cluster.rb

@@ -1,11 +1,12 @@
 require 'chef/provisioning/aws_driver/aws_provider'
 
 class Chef::Provider::AwsCacheCluster < Chef::Provisioning::AWSDriver::AWSProvider
+  provides :aws_cache_cluster
 
   protected
 
   def create_aws_object
-    converge_by "create new Elasticache Cluster #{new_resource.name} in #{region}" do
+    converge_by "create ElastiCache cluster #{new_resource.name} in #{region}" do
       driver.create_cache_cluster(desired_options)
     end
   end
@@ -23,7 +24,7 @@ class Chef::Provider::AwsCacheCluster < Chef::Provisioning::AWSDriver::AWSProvid
   end
 
   def destroy_aws_object(cache_cluster)
-    converge_by "delete Elasticache Cluster #{new_resource.name} in #{region}" do
+    converge_by "delete ElastiCache cluster #{new_resource.name} in #{region}" do
       driver.delete_cache_cluster(
         cache_cluster_id: cache_cluster[:cache_cluster_id]
       )

data/lib/chef/provider/aws_cache_replication_group.rb

@@ -1,21 +1,22 @@
 require 'chef/provisioning/aws_driver/aws_provider'
 
 class Chef::Provider::AwsCacheReplicationGroup < Chef::Provisioning::AWSDriver::AWSProvider
-
+  provides :aws_cache_replication_group
+  
   protected
 
   def create_aws_object
-    converge_by "create new Elasticache Replication Group #{new_resource.name} in #{region}" do
+    converge_by "create ElastiCache replication group #{new_resource.name} in #{region}" do
       driver.create_replication_group(desired_options)
     end
   end
 
   def update_aws_object(cache_replication_group)
-    Chef::Log.warn('Updating Elasticache Replication Groups is currently unsupported')
+    Chef::Log.warn('Updating ElastiCache replication groups is currently unsupported')
   end
 
   def destroy_aws_object(cache_replication_group)
-    converge_by "delete Elasticache Replication group #{new_resource.name} in #{region}" do
+    converge_by "delete ElastiCache replication group #{new_resource.name} in #{region}" do
       driver.delete_replication_group(
         replication_group_id: cache_replication_group[:replication_group_id]
       )

data/lib/chef/provider/aws_cache_subnet_group.rb

@@ -1,25 +1,26 @@
 require 'chef/provisioning/aws_driver/aws_provider'
 
 class Chef::Provider::AwsCacheSubnetGroup < Chef::Provisioning::AWSDriver::AWSProvider
-
+  provides :aws_cache_subnet_group
+  
   protected
 
   def create_aws_object
-    converge_by "create new Elasticache Subnet Group #{new_resource.name} in #{region}" do
+    converge_by "create ElastiCache subnet group #{new_resource.name} in #{region}" do
       driver.create_cache_subnet_group(desired_options)
     end
   end
 
   def update_aws_object(cache_subnet_group)
     if update_required?(cache_subnet_group)
-      converge_by "update Elasticache Subnet Group #{new_resource.name} in #{region}" do
+      converge_by "update ElastiCache subnet group #{new_resource.name} in #{region}" do
         driver.modify_cache_subnet_group(desired_options)
       end
     end
   end
 
   def destroy_aws_object(cache_subnet_group)
-    converge_by "delete Elasticache Subnet Group #{new_resource.name} in #{region}" do
+    converge_by "delete ElastiCache subnet group #{new_resource.name} in #{region}" do
       driver.delete_cache_subnet_group(
         cache_subnet_group_name: cache_subnet_group[:cache_subnet_group_name]
       )

data/lib/chef/provider/aws_cloudsearch_domain.rb

@@ -0,0 +1,163 @@
+require 'chef/provisioning/aws_driver/aws_provider'
+
+class Chef::Provider::AwsCloudsearchDomain < Chef::Provisioning::AWSDriver::AWSProvider
+  provides :aws_cloudsearch_domain
+  
+  def create_aws_object
+    domain = nil # define here to ensure it is available outside of the coverge_by scope
+    converge_by "create CloudSearch domain #{new_resource.name}" do
+      domain = create_domain
+    end
+
+    update_aws_object(domain)
+  end
+
+  def destroy_aws_object(domain)
+    converge_by "delete CloudSearch domain #{new_resource.name}" do
+      cs_client.delete_domain(domain_name: new_resource.name)
+    end
+    # CloudSearch can take over 30 minutes to delete so im not adding a waiter
+    # for now
+  end
+
+  def update_aws_object(domain)
+    if update_availability_options?(domain)
+      converge_by "update availability options for CloudSearch domain #{new_resource}" do
+        update_availability_options
+      end
+    end
+
+    if update_scaling_params?(domain)
+      converge_by "update scaling parameters for CloudSearch domain #{new_resource.name}" do
+        update_scaling_parameters
+      end
+    end
+
+    if update_policy?(domain)
+      converge_by "update access policy for CloudSearch domain #{new_resource.name}" do
+        update_service_access_policy
+      end
+    end
+
+    if update_index_fields?(domain)
+      Chef::Log.warn("Updating existing index_fields not currently supported")
+    end
+  end
+
+  private
+
+  def update_availability_options?(_domain)
+    # new_resource.multi_az defaults to false so we don't need an existence check
+    new_resource.multi_az != availability_options
+  end
+
+  def update_scaling_params?(domain)
+    if new_resource.partition_count || new_resource.replication_count || new_resource.instance_type
+      # We don't want to change scaling parameters that the user
+      # didn't specify. Thus, we compare on a key-by-key basis.  Only
+      # user-specified keys show up in desired_scaling_parameters
+      actual_scaling_parameters = scaling_parameters(domain)
+      desired_scaling_parameters.each do |key, value|
+        return true if value != actual_scaling_parameters[key]
+      end
+      false
+    else
+      false
+    end
+  end
+
+  def update_policy?(_domain)
+    if !new_resource.access_policies.nil?
+      new_resource.access_policies != access_policies
+    else
+      false
+    end
+  end
+
+  def update_index_fields?(domain)
+    if ! new_resource.index_fields.nil?
+      new_resource.index_fields != index_fields
+    else
+      false
+    end
+  end
+
+  def desired_scaling_parameters
+    ret = {}
+    ret[:desired_partition_count] = new_resource.partition_count if new_resource.partition_count
+    ret[:desired_replication_count] = new_resource.replication_count if new_resource.replication_count
+    ret[:desired_instance_type] =  new_resource.instance_type if new_resource.instance_type
+    ret
+  end
+
+  #
+  # API Update Functions
+  #
+  # The following functions all make changes to our domain.  Unlike
+  # other AWS APIs we don't have a single modify function for this
+  # domain.  Rather, updates our split up over a number of different
+  # API requestsion.
+  #
+  def create_domain
+    cs_client.create_domain(domain_name: new_resource.name)[:domain_status]
+  end
+
+  def update_availability_options
+    cs_client.update_availability_options(domain_name: new_resource.name,
+                                          multi_az: new_resource.multi_az)
+  end
+
+  def update_scaling_parameters
+    cs_client.update_scaling_parameters(domain_name: new_resource.name,
+                                        scaling_parameters: desired_scaling_parameters)
+  end
+
+  def update_service_access_policy
+    cs_client.update_service_access_policies(domain_name: new_resource.name,
+                                             access_policies: new_resource.access_policies)
+  end
+
+  def create_index_field(field)
+    cs_client.define_index_field(domain_name: new_resource.name, index_field: field)
+  end
+
+  #
+  # API Query Functions
+  #
+  # The CloudSearch API doesn't provide all of the data about the
+  # domain's settings via the descrbe domain API.  We have to call
+  # additional endpoints to determine the current values of:
+  # availability_options, scalability_parameters, index_fields, and
+  # access_policies
+  #
+  def availability_options
+    get_option(:availability_options)
+  end
+
+  def scaling_parameters(object)
+    o = get_option(:scaling_parameters)
+    o.merge(desired_instance_type: object[:search_instance_type])
+  end
+
+  def access_policies
+    get_option(:service_access_policies, :access_policies)
+  end
+
+  def index_fields
+    cs_client.describe_index_fields(domain_name: new_resource.name)[:index_fields]
+  end
+
+  def get_option(option_name, key=nil)
+    opt = cs_client.send("describe_#{option_name}".to_sym,
+                         {domain_name: new_resource.name})[key || option_name]
+    if ! opt[:status][:pending_deletion]
+      opt[:options]
+    else
+      nil
+    end
+  end
+
+  def cs_client
+    @cs_client ||= new_resource.driver.cloudsearch(new_resource.cloudsearch_api_version)
+  end
+end