cm 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: edae52b5afcceb98e314ad3cf88c6e85e92936b3
-  data.tar.gz: 3b7f5440d1f088d7010ff5e9f9eee485ea3bdebc
+  metadata.gz: 056ec04a3593572a5b3d8ebf419d51d43a0682f6
+  data.tar.gz: f83af1f80f2ffa7ee0a66ebc16e92e66cddb7ec3
 SHA512:
-  metadata.gz: 4f3107e50edb0d39d060650f64e136dd527723da8652c49ce0c6454f36bd220b649df1a7360462de6ea074608dd20d53f878c24f4bdfa20bb33a0813af202d4d
-  data.tar.gz: 8af5ebdb2a390a1d1a9589a99dff19bf421c75b49aa2c28628abd5d3f988540e090d6f94008c688e061a9ee9d557deb623fc4be7f63d722277743b1ed7bd3631
+  metadata.gz: 01caec2831c096f83f8bde1d2823aaaaf126a651171c6ea8e28946d34b52aa1e41112726ca12aa1fd9cab7719e5e0c3536513926c438d397177768856bbdc3e7
+  data.tar.gz: 8fd25cc3a79ee4833335bb2c21b7fc8388bcb706d1a634b93726f438380aa419fc3aeb876da09ad7b112dddc2207773bb429d164094e969648d5b7eb5739829c

data/.gitignore

@@ -11,6 +11,9 @@ pkg
 # Gem builds
 *.gem
 
+# Vagrant
+.vagrant
+
 # Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
 #
 # * Create a file at ~/.gitignore

data/.gitmodules

@@ -1,3 +1,6 @@
 [submodule "bootstrap"]
 	path = bootstrap
 	url = https://github.com/adrianwebb/cm-bootstrap.git
+[submodule "toolbox"]
+	path = toolbox
+	url = https://github.com/adrianwebb/gem-toolbox.git

data/Gemfile

@@ -2,9 +2,11 @@ source "http://rubygems.org"
 
 gem "nucleon", "~> 0.2"
 gem "docker-api", "~> 1.24"  # Compatible with Docker API version 1.16
+gem "nokogiri", "= 1.5.11"
+gem "fog-aws", "~> 0.8"
 
 group :development do
-  gem "bundler", "~> 1.10"
+  gem "bundler", ">= 1.7"
   gem "jeweler", "~> 2.0"
   gem "rspec", "~> 3.4"
   gem "rdoc", "~> 4.2"

data/Gemfile.lock

@@ -20,34 +20,48 @@ GEM
       timers (>= 4.1.1)
     celluloid-supervision (0.20.5)
       timers (>= 4.1.1)
-    childprocess (0.5.8)
+    childprocess (0.5.9)
       ffi (~> 1.0, >= 1.0.11)
     deep_merge (1.0.1)
-    descendants_tracker (0.0.4)
-      thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.2.5)
     docker-api (1.24.1)
       excon (>= 0.38.0)
       json
     erubis (2.7.0)
     excon (0.45.4)
-    faraday (0.9.2)
-      multipart-post (>= 1.2, < 3)
+    faraday (0.8.11)
+      multipart-post (~> 1.2.0)
     ffi (1.9.10)
+    fog-aws (0.8.1)
+      fog-core (~> 1.27)
+      fog-json (~> 1.0)
+      fog-xml (~> 0.1)
+      ipaddress (~> 0.8)
+    fog-core (1.35.0)
+      builder
+      excon (~> 0.45)
+      formatador (~> 0.2)
+    fog-json (1.0.2)
+      fog-core (~> 1.0)
+      multi_json (~> 1.10)
+    fog-xml (0.1.2)
+      fog-core
+      nokogiri (~> 1.5, >= 1.5.11)
+    formatador (0.2.5)
     git (1.2.9.1)
-    github_api (0.13.0)
-      addressable (~> 2.3)
-      descendants_tracker (~> 0.0.4)
-      faraday (~> 0.8, < 0.10)
-      hashie (>= 3.4)
-      multi_json (>= 1.7.5, < 2.0)
-      nokogiri (~> 1.6.6)
+    github_api (0.10.1)
+      addressable
+      faraday (~> 0.8.1)
+      hashie (>= 1.2)
+      multi_json (~> 1.4)
+      nokogiri (~> 1.5.2)
       oauth2
     hashie (3.4.3)
     highline (1.7.8)
     hitimes (1.2.3)
     i18n (0.7.0)
-    jeweler (2.0.1)
+    ipaddress (0.8.2)
+    jeweler (2.0.0)
       builder
       bundler (>= 1.0)
       git (>= 1.2.5)
@@ -59,14 +73,12 @@ GEM
     json (1.8.3)
     jwt (1.5.2)
     log4r (1.1.10)
-    mini_portile2 (2.0.0)
     multi_json (1.11.2)
     multi_xml (0.5.5)
-    multipart-post (2.0.0)
+    multipart-post (1.2.0)
     netrc (0.11.0)
-    nokogiri (1.6.7)
-      mini_portile2 (~> 2.0.0.rc2)
-    nucleon (0.2.15)
+    nokogiri (1.5.11)
+    nucleon (0.2.16)
       celluloid (~> 0.17)
       childprocess (~> 0.5)
       deep_merge (~> 1.0)
@@ -89,7 +101,7 @@ GEM
       sawyer (~> 0.6.0, >= 0.5.3)
     rack (1.6.4)
     rake (10.4.2)
-    rdoc (4.2.0)
+    rdoc (4.2.1)
       json (~> 1.4)
     rspec (3.4.0)
       rspec-core (~> 3.4.0)
@@ -109,7 +121,6 @@ GEM
       addressable (~> 2.3.5)
       faraday (~> 0.8, < 0.10)
     sshkey (1.8.0)
-    thread_safe (0.3.5)
     timers (4.1.1)
       hitimes
 
@@ -117,12 +128,11 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.10)
+  bundler (>= 1.7)
   docker-api (~> 1.24)
+  fog-aws (~> 0.8)
   jeweler (~> 2.0)
+  nokogiri (= 1.5.11)
   nucleon (~> 0.2)
   rdoc (~> 4.2)
   rspec (~> 3.4)
-
-BUNDLED WITH
-   1.10.6

data/VERSION

@@ -1 +1 @@
-0.1.6
+0.1.8

data/cm.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: cm 0.1.6 ruby lib
+# stub: cm 0.1.8 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "cm"
-  s.version = "0.1.6"
+  s.version = "0.1.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Adrian Webb"]
-  s.date = "2015-12-27"
+  s.date = "2016-01-18"
   s.description = "\nPluggable cloud management framework that provides a simple foundation for\ndeploying and destroying enterprise ready cloud environments and components\nthat integrate; cloud provider components and services, cloud orchestration\nand configuration management tools, and continuous integration and delivery\npipelines.\n"
   s.email = "adrian.webb@gsa.gov"
   s.executables = ["cm"]
@@ -35,9 +35,8 @@ Gem::Specification.new do |s|
     "lib/CM/plan/default.rb",
     "lib/CM/resource/AWS.rb",
     "lib/CM/resource/BOSH.rb",
-    "lib/CM/resource/MicroBOSH.rb",
     "lib/CM/resource/concourse.rb",
-    "lib/CM/resource/keypair.rb",
+    "lib/CM/resource/plan.rb",
     "lib/CM/resource/variables.rb",
     "lib/CM/sequence/default.rb",
     "lib/cm.rb",
@@ -62,13 +61,16 @@ Gem::Specification.new do |s|
     "lib/nucleon/action/plan/destroy.rb",
     "lib/nucleon/action/resource/run.rb",
     "locales/en.yml",
-    "spec/spec_helper.rb"
+    "spec/spec_helper.rb",
+    "toolbox/LICENSE.txt",
+    "toolbox/README.txt",
+    "toolbox/lib/shell/LICENSE.txt"
   ]
   s.homepage = "http://github.com/adrianwebb/cm"
   s.licenses = ["Apache License, Version 2.0"]
   s.rdoc_options = ["--title", "CM (Cloud Manager)", "--main", "README.rdoc", "--line-numbers"]
   s.required_ruby_version = Gem::Requirement.new(">= 1.9.1")
-  s.rubygems_version = "2.4.5"
+  s.rubygems_version = "2.4.8"
   s.summary = "Pluggable cloud management framework that provides a simple foundation for deploying and destroying enterprise ready cloud environments and components"
 
   if s.respond_to? :specification_version then
@@ -77,14 +79,18 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<nucleon>, ["~> 0.2"])
       s.add_runtime_dependency(%q<docker-api>, ["~> 1.24"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.10"])
+      s.add_runtime_dependency(%q<nokogiri>, ["= 1.5.11"])
+      s.add_runtime_dependency(%q<fog-aws>, ["~> 0.8"])
+      s.add_development_dependency(%q<bundler>, [">= 1.7"])
       s.add_development_dependency(%q<jeweler>, ["~> 2.0"])
       s.add_development_dependency(%q<rspec>, ["~> 3.4"])
       s.add_development_dependency(%q<rdoc>, ["~> 4.2"])
     else
       s.add_dependency(%q<nucleon>, ["~> 0.2"])
       s.add_dependency(%q<docker-api>, ["~> 1.24"])
-      s.add_dependency(%q<bundler>, ["~> 1.10"])
+      s.add_dependency(%q<nokogiri>, ["= 1.5.11"])
+      s.add_dependency(%q<fog-aws>, ["~> 0.8"])
+      s.add_dependency(%q<bundler>, [">= 1.7"])
       s.add_dependency(%q<jeweler>, ["~> 2.0"])
       s.add_dependency(%q<rspec>, ["~> 3.4"])
       s.add_dependency(%q<rdoc>, ["~> 4.2"])
@@ -92,7 +98,9 @@ Gem::Specification.new do |s|
   else
     s.add_dependency(%q<nucleon>, ["~> 0.2"])
     s.add_dependency(%q<docker-api>, ["~> 1.24"])
-    s.add_dependency(%q<bundler>, ["~> 1.10"])
+    s.add_dependency(%q<nokogiri>, ["= 1.5.11"])
+    s.add_dependency(%q<fog-aws>, ["~> 0.8"])
+    s.add_dependency(%q<bundler>, [">= 1.7"])
     s.add_dependency(%q<jeweler>, ["~> 2.0"])
     s.add_dependency(%q<rspec>, ["~> 3.4"])
     s.add_dependency(%q<rdoc>, ["~> 4.2"])

data/lib/CM/batch/celluloid.rb

@@ -25,12 +25,32 @@ class Celluloid < Nucleon.plugin_class(:CM, :batch)
 
   def execute_parallel(operation)
     values = []
+    timeouts = []
+
     resources.each do |resource|
       values << ::Celluloid::Future.new(resource) do
-        resource.execute(operation)
+        success = true
+        begin
+          timeouts << ( resource.respond_to?(:timeout) ? resource.timeout : nil )
+
+          resource.execute(operation)
+          success = resource.status == code.success
+
+        rescue => error
+          logger.error("Resource #{resource.id} #{operation} experienced an error:")
+          logger.error(error.inspect)
+          logger.error(error.message)
+          logger.error(Nucleon::Util::Data.to_yaml(error.backtrace))
+
+          error('resource_execution', { :id => resource.id, :operation => operation, :message => error.message })
+          success = false
+        end
+        success
       end
     end
-    values = values.map { |future| future.value }
+    values = values.each_with_index.map do |future, index|
+      future.value(timeouts[index])
+    end
     !values.include?(false)
   end
 end

data/lib/CM/resource/AWS.rb

@@ -8,6 +8,9 @@ class AWS < Nucleon.plugin_class(:CM, :docker_resource)
 
   def normalize(reload)
     super
+
+    codes :aws_request_failed
+
     yield if block_given?
   end
 
@@ -21,26 +24,199 @@ class AWS < Nucleon.plugin_class(:CM, :docker_resource)
   #-----------------------------------------------------------------------------
   # Property accessors / modifiers
 
+  def template
+    settings[:template].to_sym
+  end
+
+  #---
+
+  def manifest_config
+    @manifest_config ||= plan.manifest_config
+  end
+
+  #---
+
+  def compute
+    @compute ||= init_compute
+  end
+
+  #---
+
+  def key_file(name)
+    File.join(plan.key_directory, "#{name}.pem")
+  end
+
   #-----------------------------------------------------------------------------
   # Operations
 
-  def operation_deploy
-    super do
-      success = true
+  def create_resource
+    super do |data|
+      if template == :keypair
+        create_keypair(parameters[:Name], data)
+      else
+        #info('create_stack', { :id => id, :name => template, :prefix => false })
+        #create_stack(template, parameters, data)
+      end
     end
   end
 
-  #---
+  def retrieve_resource
+    resource = nil
+    if template == :keypair
+      resource = fetch_keypair(parameters[:Name])
+    else
+      #resource = fetch_stack(template)
+      #dbg(resource, 'retrieval results')
+    end
+    resource
+  end
+
+  def update_resource
+    super do |data|
+      if template == :keypair
+        update_keypair(parameters[:Name], data)
+      else
+        #info('update_stack', { :id => id, :name => template, :prefix => false })
+        #update_stack(template, parameters, data)
+      end
+    end
+  end
+
+  def delete_resource
+    super do |data|
+      if template == :keypair
+        delete_keypair(parameters[:Name], data)
+      else
+        #info('delete_stack', { :id => id, :name => template, :prefix => false })
+        #delete_stack(template, data)
+      end
+    end
+  end
+
+  #-----------------------------------------------------------------------------
+  # Keypair related functionality
+
+  def fetch_keypair(name, reset = false)
+    if reset || !@keypair
+      begin
+        aws_keypair = nil
+
+        result = compute.describe_key_pairs({ 'key-name' => [name] })
+        aws_keypair = result.body['keySet'].first['keyFingerprint'] if result.body['keySet'].length == 1
+
+        if aws_keypair
+          @keypair = {
+            :name => name.to_sym,
+            :fingerprint => aws_keypair,
+            :file => key_file(name)
+          }
+        end
+      rescue => error
+        # Placeholder for logging in the future
+        raise error
+      end
+    end
+    @keypair
+  end
 
-  def operation_destroy
-    super do
-      success = true
+  def create_keypair(name, data, suppress_info = false)
+    begin
+      info('create_keypair', { :id => id, :name => parameters[:Name] }) unless suppress_info
+
+      private_key_file = key_file(name)
+      result = compute.create_key_pair(name)
+
+      Nucleon::Util::Disk.write(private_key_file, result.body['keyMaterial'])
+      FileUtils.chmod(0600, private_key_file)
+
+      aws_keypair = fetch_keypair(name, true)
+      data[:Fingerprint] = aws_keypair[:fingerprint]
+      data[:Content] = result.body['keyMaterial']
+      data[:File] = aws_keypair[:file]
+
+    rescue => error
+      myself.status = code.aws_request_failed
+      raise error
+    end
+  end
+
+  def update_keypair(name, data)
+    begin
+      info('check_keypair', { :id => id, :name => parameters[:Name] })
+
+      aws_keypair = fetch_keypair(name)
+      private_key_file = aws_keypair[:file]
+      update = false
+
+      if File.exist?(private_key_file)
+        local_keypair = `openssl pkcs8 -in #{private_key_file} -inform PEM -outform DER -topk8 -nocrypt | openssl sha1 -c`.strip.sub(/^[^\s]+\s+/, '')
+
+        if aws_keypair[:fingerprint] != local_keypair
+          File.delete(private_key_file)
+          update = true
+        end
+      else
+        update = true
+      end
+
+      if update
+        info('update_keypair', { :id => id, :name => parameters[:Name] })
+
+        compute.delete_key_pair(name)
+        create_keypair(name, data, true)
+      end
+    rescue => error
+      myself.status = code.aws_request_failed
+      raise error
+    end
+  end
+
+  def delete_keypair(name, data)
+    begin
+      info('delete_keypair', { :id => id, :name => parameters[:Name] })
+
+      if aws_keypair = fetch_keypair(name)
+        result = compute.delete_key_pair(name)
+        File.delete(aws_keypair[:file]) if File.exist?(aws_keypair[:file])
+      end
+    rescue => error
+      myself.status = code.aws_request_failed
+      raise error
     end
   end
 
+  #-----------------------------------------------------------------------------
+  # Stack related functionality
+
+  def fetch_stack(name, data)
+
+  end
+
+  def create_stack(name, parameters = {}, data)
+
+  end
+
+  def update_stack(name, parameters = {}, data)
+
+  end
+
+  def delete_stack(name, data)
+
+  end
+
   #-----------------------------------------------------------------------------
   # Utilities
 
+  def init_compute
+    require 'fog/aws'
+
+    Fog::Compute.new({
+      :provider => 'AWS',
+      :region => manifest_config[:aws][:Region],
+      :aws_access_key_id => manifest_config[:aws][:AccessKey],
+      :aws_secret_access_key => manifest_config[:aws][:SecretAccessKey]
+    })
+  end
 end
 end
 end