topo-provision 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9f98f28285a7ba5c4b90fb7a393dd8c8ccf45fdc
+  data.tar.gz: 6fa3d26529b683afb9b21a9be162d48a937d8f09
+SHA512:
+  metadata.gz: d42c65bdfb1750c68da96e253bfc8def52a02f7694bdb3ad3a593fb7cf7b1dc503e45f7c6c215e0b790592c330cf3a17fa7d693c8b3d274de4dc0029ba169aea
+  data.tar.gz: 1dab45430c5a965f54202ffabd2a7c347fb4d39ef2e764d269cd7ff5c2cde273dac09144fdc2b9e2730d765e680c1436175bbebc03410a40721b41a3059d7c48

data/LICENSE

@@ -0,0 +1,201 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2015 ThirdWave Insights, LLC
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,100 @@
+# Topo::Provision
+
+This utility allows you to generate chef-provisioning recipes from
+a topology JSON file. 
+
+The format is an extended version of the JSON used by the knife-topo 
+plugin. In addition, it is possible to convert from other formats. The
+only converter currently implemented is for CloudFormation.
+
+This utility IS NOT READY FOR PRODUCTION USE. It is best described as a
+"proof of concept". It has been tested with a pre-release version of 
+chef-provisioning 1.0.
+
+## Features
+
+'Nodes' in the topology JSON are generated as machine resources, if
+provisioning is specified; otherwise they are generated as chef nodes.
+
+'Node groups' in the topology JSON are generated as combinations of
+different resources, depending on the driver and options specified.
+Currently, for AWS if 'auto_scaling' is specified, it
+generates a machine_image (unless an explicit image_id) is specified;
+a launch configuration and an auto-scaling group. For vagrant, the action
+is to generate multiple machines using machine_batch.
+
+'Services' can be used to specify other sorts of resource, such as 
+a load balancer.
+
+Node attributes can use a special topo_ref to indicate when an attribute
+should be set based on attributes of another node. topo-provision will
+generate the appropriate 'lazy' evaluation of the attribute value using
+a node search, and will use the information to order the resources in the
+generated recipe appropriately.
+
+## Usage
+
+    $ topo-provision topofile.json
+    
+    $ topo-provision cloudformation_template --format=cloudformation --output-topo cloudformation.json
+    
+    $ topo-provision topofile.json --output topo_deploy.json
+    
+    $ topo-provision topofile.json --action=undeploy --output topo_undeploy.json
+    
+    $ topo-provision --help
+
+## Structure
+
+### Generators
+
+Driver Generators support a particular chef provisioning driver, e.g. vagrant,
+aws, fog. They live in `lib/topo/provision/#{driver}/` and register themselves using the
+name of the driver. They should extend the base `Topo::Provision::Generator`
+class and override its methods as needed to process the topo elements (context, node, service)
+and call the appropriate Resource Generators.
+
+Resource Generators live in `lib/topo/provision/#{driver}/generators` and
+should extend `Topo::Provision::ResourceGenerator`. They are initialized
+from a topo context, node or service element and generate the actual output 
+for the actions that are supported on the specific resource type (e.g. deploy, undeploy, stop).
+
+The Topology Generator `Topo::Provision::TopologyGenerator` is the top-level generator that calls the appropriate
+driver generators and generates a resource graph based on topo_ref dependencies, that it traverses in either
+a forward or reverse direction, depending on the actions.
+
+### Converters
+
+Converters accept a format other than topology JSON, and convert it to
+the standard topo format. They live in `lib/topo/converter` and register themselves using the
+name that should be passed in using the --format option. They should extend
+`Topo::Converter` and provide a 'convert' method that takes a data hash and returns a topo hash.
+
+
+## Known Limitations
+
+Only supports the vagrant and aws drivers.
+
+Only covers the following resources:
+* machine
+* machine_image (as part of creating a node group)
+* load_balancer
+* aws_auto_scaling_group (as part of creating a node group)
+* aws_launch_configuration (as part of creating a node group)
+
+## Future Directions
+
+Here's some things I would like to do, beyond extending current support:
+* Support network resources
+* Have generators analyse for dependencies (e.g. name refs to other
+resources) and include these in the topo ordering as well as explicit topo_refs
+* Allow users to customize behavior by providing alternative generators
+
+
+## Contributing
+
+1. Fork it ( https://github.com/christinedraper/topo-provision/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/bin/topo-provision

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+#
+# Author:: Christine Draper (<christine_draper@thirdwaveinsights.com>)
+# Copyright:: Copyright (c) 2015 ThirdWave Insights LLC
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'topo/provision/cli'
+
+Topo::Provision::CLI.new.run

data/lib/topo/converter.rb

@@ -0,0 +1,80 @@
+#
+# Author:: Christine Draper (<christine_draper@thirdwaveinsights.com>)
+# Copyright:: Copyright (c) 2015 ThirdWave Insights LLC
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# The converter class converts data in a given format into raw "topo" format data
+#
+
+require 'topo/utils/parsegen'
+
+module Topo
+  class Converter
+    
+    include Topo::ParseGen
+
+    attr_accessor :topology
+
+    # Converters for each format
+    @@converter_classes = {}
+
+    def self.register_converter(format, class_name)
+      @@converter_classes[format] = class_name
+    end
+
+    self.register_converter("default", self.name)
+    self.register_converter("topo", self.name)
+
+    # Get the right converter for the input format
+    def self.converter(format)
+      converter_class = @@converter_classes[format]
+
+      unless converter_class
+        begin
+          require "topo/converter/#{format}/converter"
+          converter_class = @@converter_classes[format]
+        rescue LoadError
+          STDERR.puts("#{format} is not a known format for the topology file")
+          exit 1
+        end
+      end
+
+      Object::const_get(converter_class).new(format)
+    end
+
+    def self.convert(data, format)
+      converter = self.converter(format)
+      converter.convert(data)
+    end
+
+    attr_accessor :input
+    
+    def initialize(format, data=nil)
+      @format = format
+      @input = data
+      @output = { "nodes" => [], "services" => [], "network" => [], "provisioning" => {} }
+    end
+    
+    # Other format converters should override this method to convert to topo format data
+    def convert(data=nil)
+      @input = data if data
+      @output = @input
+      @output
+    end
+
+  end
+end

data/lib/topo/converter/cloudformation/converter.rb

@@ -0,0 +1,219 @@
+#
+# Author:: Christine Draper (<christine_draper@thirdwaveinsights.com>)
+# Copyright:: Copyright (c) 2015 ThirdWave Insights LLC
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+#
+
+module Topo
+  module Provision
+    class CloudFormationConverter < Topo::Converter
+
+      Topo::Converter.register_converter("cloudformation", self.name)
+      
+      def convert(data)
+        @input = data if data
+        
+        @output['name'] = @input['name'] || "CloudFormation"
+        
+        create_provisioning
+
+        @input['Resources'].each do |name, resource|
+          convert_resource(name, resource)
+        end
+        
+        @output
+      end
+      
+
+      def create_provisioning
+        @output['provisioning'] = {'driver' => "aws", 'machine_options'=> {  }}
+      end
+      
+      FIELDS = {
+        :boolean => {
+          :bootstrap_options => %w[disable_api_termination associate_public_ip_address 
+            ebs_optimized dedicated_tenancy monitoring monitoring_enabled ],
+          :network_interface => %w[ requester_managed source_dest_check delete_on_termination ],
+          :block_device_mapping_ebs => %w[delete_on_termination]
+
+        },
+        :integer => {
+          :listener => %w[instance_port load_balancer_port],
+          :network_interface => %w[ device_index ],
+          :block_device_mapping_ebs => %w[volume_size iops],
+          :auto_scaling_group => %w[max_size min_size desired_capacity cooldown default_cooldown health_check_grace_period]
+        },
+        :maptov1 => {
+          :listener => { "load_balancer_port" => "port" },
+          :bootstrap_options => { "monitoring" => "monitoring_enabled" },
+          :launch_config => { "monitoring" => "detailed_instance_monitoring" }
+        }
+      }
+
+      # CloudFormation keys are camelcase - convert to underscore
+      # CloudFormation values are all strings - convert to proper type
+      # CloudFormation is using AWS V2, chef-provisioning-aws is using AWS V1 
+      def convert_resource(name, resource)
+        
+        props = value_from_path(resource, ["Properties"]) || {}
+        props = keys_to_underscore props
+         
+        case resource['Type']
+        when "AWS::EC2::Instance"
+          @output['nodes'] << convert_instance(name, resource)
+
+        when "AWS::ElasticLoadBalancing::LoadBalancer"
+          listeners = value_from_path(props, ["listeners"]) || []
+          props["listeners"] = listeners.map{|listener|             
+            fields_to_v1(:listener, fields_to_i(:listener, keys_to_underscore(listener)))
+          } 
+          @output['services'] << { "name" => name, "type" => "load_balancer", 
+            "provisioning" => { "load_balancer_options" => props } }
+
+        when "AWS::AutoScaling::AutoScalingGroup"
+          @output['nodes'] << convert_auto_scaling_group(name, resource)
+        end
+      end
+      
+      def convert_instance(name, resource)
+
+        instance = { 
+          "name" => name,
+          "provisioning" => { "machine_options" => {} }
+        }
+          
+        props = value_from_path(resource, ["Properties"]) || {}
+        props = keys_to_underscore props
+        props = fields_to_v1(:bootstrap_options, fields_to_bool(:bootstrap_options, props))
+        
+        # Need to convert object arrays (TODO helpers)
+        if props["tags"]
+          props["tags"].each_with_index do | tag, index | 
+            props["tags"][index] = keys_to_underscore(tag) 
+            STDERR.puts "WARN: Setting AWS tags on instances is not supported yet"
+            instance["provisioning"]["tags"] = props["tags"]
+            props.delete("tags")
+          end
+        end
+        if props["network_interfaces"]
+          props["network_interfaces"].each_with_index do | network_interface, index | 
+            temp = fields_to_i(:network_interface, keys_to_underscore(network_interface))
+            temp = fields_to_bool(:network_interface, temp) 
+            props["network_interfaces"][index] = temp
+            # if one of the network interfaces had associate_public_ip_address: true, then
+            # set that in bootstrap
+            if temp['associate_public_ip_address']
+              props['associate_public_ip_address'] = temp['associate_public_ip_address']
+            end
+          end
+          STDERR.puts "WARN: Setting network_interfaces on instances is not supported"
+          instance["provisioning"]["network_interfaces"] = props["network_interfaces"]
+          props.delete("network_interfaces")
+        end
+        
+        instance["provisioning"]["machine_options"]["bootstrap_options"] = props
+        
+        instance
+        
+      end
+      
+    def convert_auto_scaling_group(name, resource)
+  
+      node = { 
+        "name" => name,
+        "provisioning" => { "machine_options" => {}, "node_group" => { "auto_scaling" => {}} }       
+      }
+      # find asg properties 
+      asg_props = value_from_path(resource, %w[Properties]).clone || {}
+      asg_props = fields_to_i(:auto_scaling_group, keys_to_underscore(asg_props))
+      asg_props.delete("launch_configuration_name")
+      asg_props.delete("load_balancer_names")
+      %w[availability_zones load_balancers].each do |key|
+        node["provisioning"]["node_group"]["auto_scaling"][key] = asg_props[key] if asg_props.key? key
+        asg_props.delete(key)
+      end 
+      node["provisioning"]["node_group"]["auto_scaling"]["group_options"] = asg_props
+                
+      # find the launch config
+      lc_name = value_from_path(resource, %w[Properties LaunchConfigurationName Ref])
+      lc_props = keys_to_underscore(value_from_path(@input, ['Resources', lc_name, 'Properties'])) || {}
+      lc_props = fields_to_bool(:bootstrap_options, lc_props)
+      node["provisioning"]["node_group"]["auto_scaling"]["launch_configuration_options"] = lc_props
+      
+      # find the load balancers
+      lb_refs = value_from_path(resource, %w[Properties LoadBalancerNames]) || []
+      load_balancers = lb_refs.map{|lb_ref| lb_ref['Ref'] }
+      # TODO should this be a topo ref instead? - needs to pick up load balancer name
+      if load_balancers.length > 0
+        node["provisioning"]["node_group"]["auto_scaling"]["load_balancers"] = load_balancers
+      end
+      
+      node
+      
+    end
+      
+      
+
+      def camel_to_underscore(str)
+        str.gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+          gsub(/([a-z\d])([A-Z])/,'\1_\2').
+          downcase
+      end
+
+      def keys_to_underscore(hash)
+        converted = Hash[hash.map{|key, val|
+          if (val.kind_of? Hash)
+            [ camel_to_underscore(key), keys_to_underscore(val) ]
+          else
+            [camel_to_underscore(key), val]
+          end
+        }]
+      end
+    
+      # 
+      def fields_to_i(field_name, hash)
+        FIELDS[:integer][field_name].each do | key |
+          hash[key] = hash[key].to_i if hash.key? key
+        end
+       
+        hash
+      end
+      
+    def fields_to_bool(field_name, hash)
+      FIELDS[:boolean][field_name].each do | key |
+        if hash.key? key
+          hash[key] = (hash[key].downcase == "true")
+        end
+      end
+      hash
+    end
+    
+  def fields_to_v1(field_name, hash)
+    FIELDS[:maptov1][field_name].each do | key, value |
+     if hash.key? key
+       hash[value] = hash[key]
+       hash.delete(key)
+     end
+    end
+   
+    hash
+end
+
+    end
+  end
+end