geoengineer 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d7ba4efcd817f80f7f9f1cd14b0605e3b71cfc9c
-  data.tar.gz: 596d587ef7bbede66b50e5b72570bed3305e43a2
+  metadata.gz: ca09f8d5b8d93ce13661980ff56dd0e28763ecbf
+  data.tar.gz: 4d9e40dfd4402a197a4bab1fb3a184966045fd24
 SHA512:
-  metadata.gz: c7bf1e7081f46e7b6d2468a37c5fcca8fb29a783f3802d2d5f74e19dafb9120f64abcc5f61ecb5f9af8df67cdfa39aa9dd60b6aab96bf158c95393067641a812
-  data.tar.gz: 2566e1bf5352d9aa9b54d0d69bf86afec722685d1856a04566ee14ba77ef7cc73394260e5742672f218338796b23038deb20bf99cf817ba9da9fefcb3b208be7
+  metadata.gz: d99a2fe4223c269d192a8c662626fba2aacddc813ee12fc03fe1a58179a4e6ab5f2a38c21d58cb8aa12ca1e53097906cee497b4853e8ce407003c9890eaf1f7e
+  data.tar.gz: cb63c4c1a257d390ee22f992a725adb180e8a64d9374875b8cc185efb35b9114522248412cf60db060d7128fde5f7e3c6a563bdedee5b0b7be88c0381ccd695d

checksums.yaml.gz.sig

@@ -0,0 +1,5 @@
+c�o���ۋԅ���\���β�
+�*���j��'��V$A�� �;)�p�*_4N�9qx}�
+��b{��[�������
+sr�-j��f>&��q��T_}Þ�dV�$���"[�:�J��b���Da���oQ�ŽM���
+��{����b��@�I�����QW�ņz%��N��A�GU ��:��)��1 �$[

data/README.md

@@ -1,8 +1,8 @@
 # GeoEngineer
 
-<a href="https://commons.wikimedia.org/wiki/File:Mantle_of_Responsibility.png"><img src="./assets/mantle.png" align="right" alt="Mantle_of_Responsibility" /></a>
+[![CircleCI](https://circleci.com/gh/coinbase/geoengineer.svg?style=shield)](https://circleci.com/gh/coinbase/geoengineer)
 
-*[🚀 Coinbase is looking for DevOps and Software Engineers](http://grnh.se/gri162)*
+<a href="https://commons.wikimedia.org/wiki/File:Mantle_of_Responsibility.png"><img src="./assets/mantle.png" align="right" alt="Mantle_of_Responsibility" /></a>
 
 GeoEngineer provides a Ruby DSL and command line tool (`geo`) to *codeify*  then plan and execute changes to cloud resources.
 
@@ -25,13 +25,13 @@ Instructions to install Terraform can be found [here](https://www.terraform.io/d
 
 ### Install Ruby
 
-Instruction to install Ruby can be found [here](https://www.ruby-lang.org/en/documentation/installation/).
+Instructions to install Ruby can be found [here](https://www.ruby-lang.org/en/documentation/installation/).
 
 ### Install GeoEngineer
 
 ```
-gem build geoengineer.gempsec
-gem install geoengineer
+gem cert --add <(https://raw.githubusercontent.com/coinbase/geoengineer/master/certs/geoengineer-gem.pem)
+gem install geoengineer --trust-policy HighSecurity
 ```
 
 Test it is installed correctly with:

data/lib/geoengineer/cli/geo_cli.rb

@@ -139,7 +139,6 @@ class GeoCLI
       throw "Environment not set" unless @environment
 
       @environment.execute_lifecycle(:before, action_name.to_sym)
-
       errs = @environment.errors.flatten.sort
       return print_validation_errors(errs) unless errs.empty?
 
@@ -164,7 +163,7 @@ class GeoCLI
     end
   end
 
-  def gloabl_options
+  def global_options
     global_option('-e', '--environment <name>', "Environment to use")
 
     @verbose = true
@@ -186,15 +185,15 @@ class GeoCLI
 
   def run
     program :name, 'GeoEngineer'
-    program :version, '0.0.1'
+    program :version, GeoEngineer::VERSION
     program :description, 'GeoEngineer will help you Terraform your resources'
     always_trace!
 
     # check terraform installed
     return puts "Please install terraform" unless terraform_installed?
 
-    # gloabl_options
-    gloabl_options
+    # global_options
+    global_options
 
     # Add commands
     plan_cmd

data/lib/geoengineer/cli/status_command.rb

@@ -39,6 +39,11 @@ module GeoCLI::StatusCommand
     puts Terminal::Table.new({ rows: rows })
   end
 
+  def status_types(options)
+    return options.resources.split(',') if options.resources
+    environment.status_types ? environment.status_types : default_status_types
+  end
+
   def default_status_types
     [
       "aws_security_group",
@@ -77,7 +82,7 @@ module GeoCLI::StatusCommand
   def status_action
     lambda do |args, options|
       type_stats = {}
-      default_status_types.each do |type|
+      status_types(options).each do |type|
         codified = @environment.codified_resources(type)
         uncodified = @environment.uncodified_resources(type)
         type_stats[type] = calculate_type_status(codified, uncodified)
@@ -94,6 +99,7 @@ module GeoCLI::StatusCommand
     command :status do |c|
       c.syntax = 'geo status [<geo_files>]'
       c.description = 'Displays the the new, managed and unmanaged resources'
+      c.option '--resources COMMA SEPERATED STRING', String, 'select resources for statuses'
       action = status_action
       c.action init_action(:status, &action)
     end

data/lib/geoengineer/environment.rb

@@ -1,3 +1,4 @@
+
 ########################################################################
 # An Environment is a group of projects, resources and attributes,
 # build to create a terraform file.
@@ -8,48 +9,64 @@
 class GeoEngineer::Environment
   include HasAttributes
   include HasResources
+  include HasProjects
+  include HasTemplates
   include HasValidations
   include HasLifecycle
 
-  attr_reader :name, :projects
+  attr_reader :name
 
   validate -> { validate_required_attributes([:region, :account_id]) }
 
   # Validate resources have unique attributes
   validate -> {
-    resources_grouped_by(&:terraform_name)
-      .select { |k, v| v.length > 1 }
-      .map { |k, v| "Non-unique type.id #{v.first.for_resource}" }
+    resources_of_type_grouped_by(&:terraform_name).map do |klass, grouped_resources|
+      grouped_resources
+        .select { |k, v| v.length > 1 }
+        .map { |k, v| "Non-unique type.id #{v.first.for_resource}" }
+    end.flatten
   }
 
   validate -> {
-    resources_grouped_by(&:_terraform_id)
-      .select { |k, v| v.length > 1 && !v.first._terraform_id.nil? }
-      .map { |k, v| "Non-unique _terraform_id #{v.first._terraform_id} #{v.first.for_resource}" }
+    resources_of_type_grouped_by(&:_terraform_id).map do |klass, grouped_resources|
+      grouped_resources
+        .select { |k, v| v.length > 1 && !v.first._terraform_id.nil? }
+        .map { |k, v| "Non-unique _terraform_id #{v.first._terraform_id} #{v.first.for_resource}" }
+    end.flatten
   }
 
   validate -> {
-    resources_grouped_by(&:_geo_id)
-      .select { |k, v| v.length > 1 }
-      .map { |k, v| "Non-unique _geo_id #{v.first._geo_id} #{v.first.for_resource}" }
+    resources_of_type_grouped_by(&:_geo_id).map do |klass, grouped_resources|
+      grouped_resources
+        .select { |k, v| v.length > 1 }
+        .map { |k, v| "Non-unique _geo_id #{v.first._geo_id} #{v.first.for_resource}" }
+    end.flatten
   }
 
-  # Validate all resources
-  validate -> { resources.map(&:errors).flatten }
-
   # Validate all projects (which validate resources)
-  validate -> { projects.map(&:errors).flatten }
+  validate -> { projects.values.map(&:errors).flatten }
 
-  before :validation, -> { self.region = self.region || ENV['AWS_REGION'] }
+  # Validate all resources
+  validate -> { all_resources.map(&:errors).flatten }
+
+  before :validation, -> { self.region ||= ENV['AWS_REGION'] }
 
   def initialize(name, &block)
     @name = name
-    @projects = []
     @outputs = []
     self.send("#{name}?=", true) # e.g. staging?
     instance_exec(self, &block) if block_given?
   end
 
+  def project(org, name, &block)
+    project = create_project(org, name, &block)
+    supported_environments = [project.environments].flatten
+    # do not add the project if the project is not supported by this environment
+    return NullObject.new unless supported_environments.include?(@name)
+
+    projects[name] = project
+  end
+
   def resource(type, id, &block)
     return find_resource(type, id) unless block_given?
     resource = create_resource(type, id, &block)
@@ -64,53 +81,35 @@ class GeoEngineer::Environment
   end
 
   def all_resources
-    reses = resources
-    @projects.each { |project| reses += project.all_resources }
-    reses
-  end
-
-  # Factory for creating projects inside an environment
-  def project(org, name, &block)
-    # do not add the project a second time
-    exists = @projects.select { |p| p.org == org && p.name == name }.first
-    return exists if exists
-
-    project = GeoEngineer::Project.new(org, name, self, &block)
-
-    supported_environments = [project.environments].flatten
-    # do not add the project if the project is not supported by this environment
-    return NullObject.new unless supported_environments.include? @name
-
-    @projects << project
-    project
+    [resources, all_template_resources, all_project_resources].flatten
   end
 
   # DOT Methods
   # Given an attribute it tries to identify a dependency and return it
   def extract_dependencies(x)
-    if x.is_a? Array
-      x.map { |y| extract_dependencies(y) }.flatten
-    elsif x.is_a?(String)
-      res = self.find_resource_by_ref(x)
+    return x.map { |y| extract_dependencies(y) }.flatten if x.is_a? Array
+    return [x] if x.is_a?(GeoEngineer::Resource)
+
+    if x.is_a?(String)
+      res = find_resource_by_ref(x)
       return [res] if res
-    elsif x.is_a?(GeoEngineer::Resource)
-      return [x]
     end
+
     []
   end
 
   def depends_on(res)
-    all_attributes = []
-    all_attributes.concat res.attributes.values
-    all_attributes.concat res.subresources.map { |sr| sr.attributes.values }.flatten
-    dependencies = Set.new(all_attributes.map { |x| extract_dependencies(x) }.flatten)
-    dependencies.delete(nil)
-    dependencies
+    all_attributes = [res.attributes.values]
+    all_attributes
+      .concat(res.subresources.map { |sr| sr.attributes.values })
+      .map { |attr| extract_dependencies(attr) }
+      .flatten
+      .compact
+      .uniq
   end
 
   def to_dot
-    str = ["digraph {"]
-    str.concat(projects.map(&:to_dot))
+    str = ["digraph {", projects.values.map(&:to_dot)]
     all_resources.each do |res|
       str << depends_on(res).map { |r| "  #{res.to_ref.inspect} -> #{r.to_ref.inspect}" }
     end
@@ -124,7 +123,8 @@ class GeoEngineer::Environment
     # Hopefully this will stop accidentally the environment
     unless self.allow_destroy
       all_resources.each { |r|
-        r.lifecycle { prevent_destroy true }
+        r.lifecycle {} unless r.lifecycle
+        r.lifecycle.prevent_destroy = true
       }
     end
 
@@ -136,7 +136,8 @@ class GeoEngineer::Environment
   def to_terraform_json
     unless self.allow_destroy
       all_resources.each { |r|
-        r.lifecycle { prevent_destroy true }
+        r.lifecycle {} unless r.lifecycle
+        r.lifecycle.prevent_destroy = true
       }
     end
 
@@ -155,6 +156,7 @@ class GeoEngineer::Environment
 
   def to_terraform_state
     reses = all_resources.select(&:_terraform_id) # _terraform_id must not be nil
+
     reses = reses.map { |r| { "#{r.type}.#{r.id}" => r.to_terraform_state() } }.reduce({}, :merge)
 
     {

data/lib/geoengineer/project.rb

@@ -5,11 +5,13 @@
 ########################################################################
 class GeoEngineer::Project
   include HasAttributes
+  include HasLifecycle
   include HasResources
+  include HasTemplates
+  include HasSubResources
   include HasValidations
 
   attr_accessor :org, :name
-  attr_reader :templates
   attr_reader :environment
 
   validate -> { environments.nil? ? "Project #{full_name} must have an environment" : nil }
@@ -19,8 +21,8 @@ class GeoEngineer::Project
     @org = org
     @name = name
     @environment = environment
-    @templates = {}
     instance_exec(self, &block) if block_given?
+    execute_lifecycle(:after, :initialize)
   end
 
   def full_id_name
@@ -40,28 +42,7 @@ class GeoEngineer::Project
   end
 
   def all_resources
-    reses = resources
-    @templates.each { |name, template| reses += template.all_resources }
-    reses
-  end
-
-  def find_template(type)
-    clazz_name = type.split('_').collect(&:capitalize).join
-    return Object.const_get(clazz_name) if Object.const_defined? clazz_name
-
-    module_clazz = "GeoEngineer::Templates::#{clazz_name}"
-    return Object.const_get(module_clazz) if Object.const_defined? module_clazz
-
-    throw "undefined template '#{type}' for '#{clazz_name}' or 'GeoEngineer::#{clazz_name}'"
-  end
-
-  def from_template(type, name, parameters = {}, &block)
-    throw "Template '#{name}' already defined for project #{full_name}" if @templates[name]
-    clazz = find_template(type)
-    template = clazz.new(name, self, parameters)
-    @templates[name] = template
-    template.instance_exec(*template.template_resources, &block) if block_given?
-    template
+    [resources, all_template_resources].flatten
   end
 
   # dot method

data/lib/geoengineer/resource.rb

@@ -17,6 +17,8 @@ class GeoEngineer::Resource
 
   attr_reader :type, :id
 
+  before :validation, :merge_project_tags
+
   validate -> { validate_required_attributes([:_geo_id]) }
 
   def initialize(type, id, &block)
@@ -93,16 +95,48 @@ class GeoEngineer::Resource
     "${#{terraform_name}.#{attribute}}"
   end
 
+  # This tries to return the terraform ID, if that is nil, then it will return the ref
+  def to_id_or_ref
+    _terraform_id || to_ref
+  end
+
+  def reset
+    reset_attributes
+    @_remote_searched = false
+    @_remote = nil
+    self
+  end
+
+  def _json_file(attribute, path, binding_obj = nil)
+    raise "file #{path} not found" unless File.file?(path)
+
+    raw = File.open(path, "rb").read
+    interpolated = ERB.new(raw).result(binding_obj)
+    escaped = interpolated.gsub("$", "$$")
+
+    # normalize JSON to prevent terraform from e.g. newlines as legitimate changes
+    normalized = _normalize_json(escaped)
+
+    send(attribute, normalized)
+  end
+
+  def _normalize_json(json)
+    h = JSON.parse(json)
+    h.to_json
+  end
+
   # REMOTE METHODS
 
   # This method will fetch the remote resource that has the same _geo_id as the codified resource.
   # This method will:
-  # 1. return nil if no resource is found
-  # 2. return an instance of Resource with the remote attributes
-  # 3. throw an error if more than one resource has the same _geo_id
+  # 1. return resource individually if class has defined how to do so
+  # 2. return nil if no resource is found
+  # 3. return an instance of Resource with the remote attributes
+  # 4. throw an error if more than one resource has the same _geo_id
   def _find_remote_resource
-    aws_resources = self.class.fetch_remote_resources()
-    matches = aws_resources.select { |r| r._geo_id == self._geo_id }
+    return GeoEngineer::Resource.build(remote_resource_params) if find_remote_as_individual?
+
+    matches = matched_remote_resource
 
     return matches.first if matches.length == 1
     return nil if matches.empty?
@@ -110,25 +144,29 @@ class GeoEngineer::Resource
     throw "ERROR:\"#{self.type}.#{self.id}\" has #{matches.length} remote resources"
   end
 
-  def self.fetch_remote_resources
-    return @_rr_cache if @_rr_cache
-    @_rr_cache = []
-    resource_hashes = _fetch_remote_resources()
+  # By default, remote resources are bulk-retrieved. In order to fetch a remote resource as an
+  # individual, the child-class over-write 'find_remote_as_individual?' and 'remote_resource_params'
+  def find_remote_as_individual?
+    false
+  end
 
-    resource_hashes.each do |res_hash|
-      @_rr_cache << GeoEngineer::Resource.new(self.type_from_class_name, res_hash['_geo_id']) {
-        # add attributes to the resource
-        res_hash.each do |k, v|
-          self[k] = v
-        end
-      }
-    end
+  def remote_resource_params
+    {}
+  end
 
-    @_rr_cache
+  def build_individual_remote_resource
+    self.class.build(remote_resource_params)
   end
 
-  def self.clear_remote_resource_cache
-    @_rr_cache = nil
+  def matched_remote_resource
+    aws_resources = self.class.fetch_remote_resources()
+    aws_resources.select { |r| r._geo_id == self._geo_id }
+  end
+
+  def self.fetch_remote_resources
+    return @_rr_cache if @_rr_cache
+    resource_hashes = _fetch_remote_resources()
+    @_rr_cache = resource_hashes.map { |res_hash| GeoEngineer::Resource.build(res_hash) }
   end
 
   # This method must be implemented for each resource type
@@ -137,6 +175,18 @@ class GeoEngineer::Resource
     throw "NOT IMPLEMENTED ERROR for #{self.name}"
   end
 
+  def self.build(resource_hash)
+    GeoEngineer::Resource.new(self.type_from_class_name, resource_hash['_geo_id']) {
+      resource_hash.each do |k, v|
+        self[k] = v
+      end
+    }
+  end
+
+  def self.clear_remote_resource_cache
+    @_rr_cache = nil
+  end
+
   # VIEW METHODS
   def short_type
     type
@@ -162,6 +212,25 @@ class GeoEngineer::Resource
     "for resource \"#{type}.#{id}\" #{in_project}"
   end
 
+  def setup_tags_if_needed
+    tags {} unless tags
+  end
+
+  def merge_project_tags
+    return unless self.project && self.project.tags && self.support_tags?
+
+    setup_tags_if_needed
+
+    self
+      .project
+      .all_tags
+      .map(&:attributes)
+      .reduce({}, :merge)
+      .each { |key, value| tags.attributes[key] ||= value }
+
+    tags
+  end
+
   # VALIDATION METHODS
   def support_tags?
     true