aws-ec2 0.3.0 → 0.4.0

data/lib/aws_ec2/template_helper.rb

@@ -1,17 +1,35 @@
 require "base64"
 require "erb"
+require "active_support/all"
 
 module AwsEc2
   module TemplateHelper
+    # auto load all the template_helpers
+    template_helper_path = File.expand_path("../template_helper", __FILE__)
+    Dir.glob("#{template_helper_path}/*").each do |path|
+      next if File.directory?(path)
+      filename = File.basename(path, '.rb')
+      class_name = filename.classify
+      instance_eval do
+        autoload class_name.to_sym, "aws_ec2/template_helper/#{filename}"
+        include const_get(class_name)
+      end
+    end
+
     def user_data(name, base64=true)
       # allow user to specify the path also
       if File.exist?(name)
         name = File.basename(name) # normalize name, change path to name
       end
       name = File.basename(name, '.sh')
-      path = "#{root}/profiles/user-data/#{name}.sh"
+      path = "#{AwsEc2.root}/app/user-data/#{name}.sh"
       result = erb_result(path)
-      result = append_ami_creation(result)
+      result = append_scripts(result)
+
+      # save the unencoded user-data script for easy debugging
+      temp_path = "/tmp/aws-ec2/user-data.txt"
+      FileUtils.mkdir_p(File.dirname(temp_path))
+      IO.write(temp_path, result)
 
       base64 ? Base64.encode64(result).strip : result
     end
@@ -30,21 +48,36 @@ module AwsEc2
     end
 
   private
-    def append_ami_creation(user_data)
-      ami = @options[:ami]
-
-      if ami
-        # assuming that the user-data script is a bash script here for simplicity
-        # TODO: add support for other types of scripts
-        # might be able to do this by wrapping all scripts in cloud-init
-        ami_creation_snippet = AwsEc2::Ami.new(ami).user_data_snippet
-        user_data += ami_creation_snippet
-      end
+    def append_scripts(user_data)
+      # assuming user-data script is a bash script for simplicity
+      script = AwsEc2::Script.new(@options)
+      user_data += script.auto_terminate if @options[:auto_terminate]
+      user_data += script.create_ami if @options[:ami_name]
       user_data
     end
 
+    # Load custom helper methods from the project repo
+    def load_custom_helpers
+      Dir.glob("#{AwsEc2.root}/app/helpers/**/*_helper.rb").each do |path|
+        filename = path.sub(%r{.*/},'').sub('.rb','')
+        module_name = filename.classify
+
+        require path
+        self.class.send :include, module_name.constantize
+      end
+
+    end
+
     def erb_result(path)
+      load_custom_helpers
       template = IO.read(path)
+
+      # Allow a way to bypass the custom ERB error handling in case
+      # the error is in the lambdagem code.
+      if ENV['DEBUG']
+        return ERB.new(template, nil, "-").result(binding)
+      end
+
       begin
         ERB.new(template, nil, "-").result(binding)
       rescue Exception => e
@@ -71,6 +104,8 @@ module AwsEc2
             printf("%#{spacing}d %s\n", line_number, line_content)
           end
         end
+
+        puts "\nIf the this error does not make sense and the error is not in the ERB template. Run the command again with DEBUG=1 to show the full lambdagem backtrace"
         exit 1 unless ENV['TEST']
       end
     end

data/lib/aws_ec2/template_helper/ami_helper.rb

@@ -0,0 +1,23 @@
+module AwsEc2::TemplateHelper::AmiHelper
+  include AwsEc2::AwsServices
+
+  # Example:
+  #
+  #   latest_ami("ruby-2.5.0_*") => ami-122
+  #
+  # Equivalent aws cli test command:
+  #
+  #   $ aws ec2 describe-images --owners self --filters="Name=name,Values=ruby-2.5.0_*" | jq '.Images | length'
+  #
+  # Returns latest ami ami
+  def latest_ami(query, owners=["self"])
+    images = ec2.describe_images(
+      owners: owners,
+      filters: [
+        {name: "name", values: [query]}
+      ]
+    ).images
+    image = images.sort_by(&:name).reverse.first
+    image.image_id
+  end
+end

data/lib/aws_ec2/template_helper/partial_helper.rb

@@ -0,0 +1,71 @@
+module AwsEc2::TemplateHelper::PartialHelper
+  def partial_exist?(path)
+    path = partial_path_for(path)
+    path = auto_add_format(path)
+    path && File.exist?(path)
+  end
+
+  # The partial's path is a relative path given without the extension and
+  #
+  # Example:
+  # Given: file in app/partials/mypartial.sh
+  # The path should be: mypartial
+  #
+  # If the user specifies the extension then use that instead of auto-adding
+  # the detected format.
+  def partial(path,vars={}, options={})
+    path = partial_path_for(path)
+    path = auto_add_format(path)
+
+    result = erb_result(path)
+    result = indent(result, options[:indent]) if options[:indent]
+    if options[:indent]
+      # Add empty line at beginning because empty lines gets stripped during
+      # processing anyway. This allows the user to call partial without having
+      # to put the partial call at very beginning of the line.
+      # This only should happen if user is using indent option.
+      ["\n", result].join("\n")
+    else
+      result
+    end
+  end
+
+  # add indentation
+  def indent(text, indentation_amount)
+    text.split("\n").map do |line|
+      " " * indentation_amount + line
+    end.join("\n")
+  end
+
+private
+  def partial_path_for(path)
+    "#{AwsEc2.root}/app/partials/#{path}"
+  end
+
+  def auto_add_format(path)
+    # Return immediately if user provided explicit extension
+    extension = File.extname(path) # current extension
+    return path if !extension.empty?
+
+    # Else let's auto detect
+    paths = Dir.glob("#{path}.*")
+
+    if paths.size == 1 # non-ambiguous match
+      return paths.first
+    end
+
+    if paths.size > 1 # ambiguous match
+      puts "ERROR: Multiple possible partials found:".colorize(:red)
+      paths.each do |path|
+        puts "  #{path}"
+      end
+      puts "Please specify an extension in the name to remove the ambiguity.".colorize(:green)
+      exit 1
+    end
+
+    # Account for case when user wants to include a file with no extension at all
+    return path if File.exist?(path) && !File.directory?(path)
+
+    path # original path if this point is reached
+  end
+end

data/lib/aws_ec2/version.rb

@@ -1,3 +1,3 @@
 module AwsEc2
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/spec/fixtures/demo_project/config/test.yml

@@ -0,0 +1,9 @@
+---
+vpc_id: vpc-123
+db_subnet_group_name: default # default db subnet group associated with default vpc
+subnets:
+  - subnet-123
+  - subnet-456
+security_group_ids:
+  - sg-123 # test-single-instance
+s3_bucket: my-bucket # for the user-data shared scripts

data/spec/fixtures/demo_project/profiles/default.yml

@@ -0,0 +1,33 @@
+---
+# image_id: ami-97785bed # Amazon Linux AMI
+image_id: ami-4fffc834 # Amazon Lambda AMI
+  # https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html
+instance_type: t2.medium
+key_name: default
+max_count: 1
+min_count: 1
+user_data:
+iam_instance_profile:
+  name: DevLinux
+# public network settings
+security_group_ids: <%= config["security_group_ids"].inspect %>
+subnet_id: <%= config["subnets"].shuffle.first %>
+block_device_mappings:
+  - device_name: /dev/xvda
+    ebs:
+      volume_size: 20
+instance_market_options:
+  market_type: spot
+  # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_LaunchTemplateSpotMarketOptionsRequest.html
+  # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EC2/Types/SpotMarketOptions.html
+  spot_options:
+    max_price: "0.018" # $0.020/hr = $14.40/mo
+                       # $0.018/hr = $12.96/mo
+    # valid combinations:
+    #   spot_instance_type: persistent
+    #   instance_interruption_behavior: hibernate
+    # or
+    #   spot_instance_type: one-time
+    # More info: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html
+    spot_instance_type: persistent
+    instance_interruption_behavior: hibernate

data/spec/lib/cli_spec.rb

@@ -7,13 +7,18 @@ require "spec_helper"
 #   $ rake clean:vcr ; time rake
 describe AwsEc2::CLI do
   before(:all) do
-    @args = "--from Tung"
+    @args = "--noop"
   end
 
   describe "aws-ec2" do
-    it "should hello world" do
-      out = execute("exe/aws-ec2 hello world #{@args}")
-      expect(out).to include("from: Tung\nHello world")
+    it "create" do
+      out = execute("exe/aws-ec2 create server #{@args}")
+      expect(out).to include("Creating EC2 instance")
+    end
+
+    it "ami" do
+      out = execute("exe/aws-ec2 ami myimage #{@args}")
+      expect(out).to include("Creating EC2 instance")
     end
   end
 end

data/spec/spec_helper.rb

@@ -1,7 +1,8 @@
 ENV["TEST"] = "1"
-
-# require "simplecov"
-# SimpleCov.start
+ENV["AWS_EC2_ENV"] = "test"
+ENV["AWS_EC2_ROOT"] = "spec/fixtures/demo_project"
+# Ensures aws api never called. Fixture home folder does not contain ~/.aws/credentails
+ENV['HOME'] = "spec/fixtures/home"
 
 require "pp"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-ec2
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-24 00:00:00.000000000 Z
+date: 2018-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -52,6 +52,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-ec2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -157,8 +199,6 @@ files:
 - aws-ec2.gemspec
 - example/profiles/default.yml
 - example/profiles/spot.yml
-- example/profiles/spot/default.yml
-- example/profiles/spot/dev.yml
 - example/profiles/user-data/dev.sh
 - exe/aws-ec2
 - lib/aws-ec2.rb
@@ -166,19 +206,26 @@ files:
 - lib/aws_ec2/aws_services.rb
 - lib/aws_ec2/cli.rb
 - lib/aws_ec2/command.rb
+- lib/aws_ec2/compile_scripts.rb
 - lib/aws_ec2/config.rb
 - lib/aws_ec2/core.rb
 - lib/aws_ec2/create.rb
+- lib/aws_ec2/create/params.rb
+- lib/aws_ec2/dotenv.rb
 - lib/aws_ec2/help.rb
+- lib/aws_ec2/help/ami.md
 - lib/aws_ec2/help/create.md
-- lib/aws_ec2/help/spot.md
 - lib/aws_ec2/help/user_data.md
+- lib/aws_ec2/hook.rb
+- lib/aws_ec2/script.rb
 - lib/aws_ec2/scripts/ami_creation.sh
-- lib/aws_ec2/spot.rb
+- lib/aws_ec2/scripts/auto_terminate.sh
 - lib/aws_ec2/template_helper.rb
-- lib/aws_ec2/user_data.rb
-- lib/aws_ec2/util.rb
+- lib/aws_ec2/template_helper/ami_helper.rb
+- lib/aws_ec2/template_helper/partial_helper.rb
 - lib/aws_ec2/version.rb
+- spec/fixtures/demo_project/config/test.yml
+- spec/fixtures/demo_project/profiles/default.yml
 - spec/lib/cli_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/tongueroo/aws-ec2
@@ -206,5 +253,7 @@ signing_key:
 specification_version: 4
 summary: Simple tool to create AWS ec2 instances
 test_files:
+- spec/fixtures/demo_project/config/test.yml
+- spec/fixtures/demo_project/profiles/default.yml
 - spec/lib/cli_spec.rb
 - spec/spec_helper.rb

data/example/profiles/spot/default.yml

@@ -1,14 +0,0 @@
----
-spot_fleet_request_config:
-  iam_fleet_role: arn:aws:iam::123456789012:role/aws-service-role/spotfleet.amazonaws.com/AWSServiceRoleForEC2SpotFleet
-  spot_price: '0.02'
-  target_capacity: 1
-  launch_specifications:
-  - iam_instance_profile:
-      arn: arn:aws:iam::123456789012:instance-profile/IAMProfileName
-    image_id: ami-97785bed
-    instance_type: t2.medium
-    key_name: default
-    security_groups:
-    - group_id: sg-111
-    subnet_id: subnet-111, subnet-222

data/example/profiles/spot/dev.yml

@@ -1,15 +0,0 @@
----
-spot_fleet_request_config:
-  iam_fleet_role: arn:aws:iam::123456789012:role/aws-service-role/spotfleet.amazonaws.com/AWSServiceRoleForEC2SpotFleet
-  spot_price: '0.017'
-  target_capacity: 1
-  launch_specifications:
-  - iam_instance_profile:
-      arn: arn:aws:iam::123456789012:instance-profile/IAMProfileName
-    image_id: ami-97785bed
-    instance_type: t2.medium
-    key_name: default
-    security_groups:
-    - group_id: sg-111
-    subnet_id: subnet-111, subnet-222
-    user_data: "<%= user_data("dev") %>"

data/lib/aws_ec2/help/spot.md

@@ -1,3 +0,0 @@
-Examples:
-
-$ aws-ec2 spot my-request

data/lib/aws_ec2/spot.rb

@@ -1,81 +0,0 @@
-require 'yaml'
-require 'active_support/core_ext/hash'
-
-module AwsEc2
-  class Spot
-    include AwsServices
-    include Util
-
-    def initialize(options)
-      @options = options
-    end
-
-    def run
-      puts "Creating spot instance fleet request..."
-      display_info
-      if @options[:noop]
-        puts "NOOP mode enabled. spot instance fleet request not created."
-        return
-      end
-
-      resp = ec2.request_spot_fleet(params)
-      puts "Spot instance fleet request created"
-    end
-
-    # params are main derived from profile files
-    def params
-      params = load_profiles("spot/#{profile_name}")
-      params = decorate_launch_template_configs(params)
-      params.deep_symbolize_keys
-    end
-
-    # Decorates the launch_template_configs:
-    #
-    #   * Ensure that a launch template version is set
-    #   * Sets the ec2 tag name
-    def decorate_launch_template_configs(params)
-      launch_template_configs = params["spot_fleet_request_config"]["launch_template_configs"]
-      return params unless launch_template_configs
-
-      # Assume only one launch_template_configs
-      # TODO: add support for multiple launch_template_configs
-      config = launch_template_configs.first
-      spec = config["launch_template_specification"]
-      version = spec["version"]
-      unless version
-        latest_version = latest_version(spec["launch_template_name"])
-      end
-
-      config["launch_template_specification"]["version"] ||= latest_version
-
-      # TODO: figure out how to override a tag for a launch template
-      # # Sets the EC2 tag name.
-      # # Replace all tag_specifications for simplicity.
-      # tag_specifications = [{
-      #     resource_type: "instance",
-      #     tags: [{ key: "Name", value: @options[:name] }],
-      #   }]
-      # config["overrides"] << ["tag_specifications"] << tag_specifications
-
-      # replace all configs
-      params["spot_fleet_request_config"]["launch_template_configs"] = [config]
-
-      params
-    end
-
-    def latest_version(launch_template_name)
-      resp = ec2.describe_launch_template_versions(launch_template_name: launch_template_name)
-      version = resp.launch_template_versions.first
-      version.version_number.to_s # request_spot_fleet expects this to be a String
-    rescue Aws::EC2::Errors::InvalidLaunchTemplateNameNotFoundException => e
-      puts e.message
-      puts "Please double check that the launch template exists"
-      exit
-    end
-
-    def display_info
-      puts "Using the following parameters:"
-      pretty_display(params)
-    end
-  end
-end