geoengineer 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dda3981b643cb0887fcdb1e794abfa4b8c6ac9c0
-  data.tar.gz: 68d052b3c9cd6a7a3f77571837cfda41b1413161
+  metadata.gz: 10a3ba58afd64f71c7acc870e588f452f26f544a
+  data.tar.gz: 50072402a6cd1b74baa75df96161f2a7d98d56b7
 SHA512:
-  metadata.gz: 8b2bae296535675da928d1decd4e9d96f448876642e953f40d50e6f4ed919ddb3cd6bebc89380a19dddf27305735d8ed2d212af29485baf42ccd582a54751432
-  data.tar.gz: 7e9eb34de6c0d0cb88d2f182ed1e618570404d30bc6c84f97c3a04bd13544e2191ba4bb82cc29e1e333f554881701f1630702072a60dea4ae005ca04d303a434
+  metadata.gz: 9999badc1ff21e9043a4f753e62a271ab98a6d09c137b948ced6802c61c32b7e68a2321aa95d65c2ab2c77f0d6d44b103df6a13567c13a0599a66dbee34db97d
+  data.tar.gz: e5e1b196d5582b70dc792c1a163ea0bf451285048c62f5a7df8de50ddab2c9e0326e7bd47106f347057396212bdb613ab9d7e14e41dc7029919400dc41d059c3

data/lib/geoengineer/cli/geo_cli.rb

@@ -2,7 +2,6 @@ require_relative '../../geoengineer'
 require 'open3'
 require 'commander'
 require 'colorize'
-require 'terminal-table'
 require 'fileutils'
 require 'json'
 require 'singleton'
@@ -32,6 +31,7 @@ class GeoCLI
   include Singleton
   include StatusCommand
   include TerraformCommands
+  include HasLifecycle
 
   attr_accessor :environment, :env_name
 
@@ -186,6 +186,13 @@ class GeoCLI
     terraform_version.exitstatus.zero?
   end
 
+  def add_commands
+    plan_cmd
+    apply_cmd
+    graph_cmd
+    status_cmd
+  end
+
   def run
     program :name, 'GeoEngineer'
     program :version, GeoEngineer::VERSION
@@ -198,11 +205,15 @@ class GeoCLI
     # global_options
     global_options
 
+    # Require any patches to the way geo works
+    if File.file?("#{Dir.pwd}/.geo.rb")
+      require_from_pwd '.geo'
+      puts "Loaded patches from .geo.rb" if @verbose
+    end
+
     # Add commands
-    plan_cmd
-    apply_cmd
-    graph_cmd
-    status_cmd
+    add_commands
+    execute_lifecycle(:after, :add_commands)
 
     # Execute the CLI
     run!

data/lib/geoengineer/cli/status_command.rb

@@ -10,33 +10,10 @@ module GeoCLI::StatusCommand
     }
   end
 
-  def status_resource_rows(reses)
-    rows = []
-    rows << :separator
-    rows << ['TerraformID', 'GeoID']
-    rows << :separator
-    reses.each do |sg|
-      g_id = sg._geo_id
-      g_id = "<<" if sg._terraform_id == sg._geo_id
-      rows << [sg._terraform_id, g_id]
-    end
-    rows << :separator
-    rows
-  end
-
-  def status_type_rows(type, codified, uncodified, stats)
-    rows = []
-
-    # Codified resources
-    rows << [{ value: "### CODIFIED #{type} ###".colorize(:green), colspan: 2, alignment: :left }]
-    rows.concat status_resource_rows(codified)
-
-    # Uncodified resources
-    rows << [{ value: "### UNCODIFIED #{type} ###".colorize(:red), colspan: 2, alignment: :left }]
-    rows.concat status_resource_rows(uncodified)
-
-    rows.concat status_rows(stats)
-    puts Terminal::Table.new({ rows: rows })
+  def resource_id_array(resources)
+    resources
+      .select { |r| !r.attributes.empty? }
+      .map { |r| r._geo_id || r._terraform_id }
   end
 
   def status_types(options)
@@ -62,35 +39,50 @@ module GeoCLI::StatusCommand
       total: 0
     }
     type_stats.each do |type, stats|
-      totals[:codified] += stats[:codified]
-      totals[:uncodified] += stats[:uncodified]
-      totals[:total] += stats[:total]
+      totals[:codified] += stats[:stats][:codified]
+      totals[:uncodified] += stats[:stats][:uncodified]
+      totals[:total] += stats[:stats][:total]
     end
     totals[:percent] = (100.0 * totals[:codified]) / totals[:total]
     totals
   end
 
-  def status_rows(stats)
-    rows = []
-    rows << ['CODIFIED'.colorize(:green), stats[:codified]]
-    rows << ['UNCODIFIED'.colorize(:red), stats[:uncodified]]
-    rows << ['TOTAL'.colorize(:blue), stats[:total]]
-    rows << ['PERCENT CODIFIED'.colorize({ mode: :bold }), format('%.2f%', stats[:percent])]
-    rows
+  def report_json(type_stats, status)
+    status[:resources] = {}
+    type_stats.each do |type, resources|
+      status[:resources][type] = {}
+      status[:resources][type][:uncodified] = resource_id_array(resources[:uncodified])
+      status[:resources][type][:codified] = resource_id_array(resources[:codified])
+    end
+    status
+  end
+
+  def type_stats(options)
+    type_stats = {}
+    status_types(options).each do |type|
+      type_stats[type] = {}
+      type_stats[type][:codified] = @environment.codified_resources(type)
+      type_stats[type][:uncodified] = @environment.uncodified_resources(type)
+      type_stats[type][:stats] = calculate_type_status(
+        type_stats[type][:codified],
+        type_stats[type][:uncodified]
+      )
+    end
+    type_stats
+  end
+
+  def only_codified(status)
+    status[:resources]
+      .select { |t, r| r[:uncodified].any? }
+      .each { |t, r| r.delete(:codified) }
   end
 
   def status_action
     lambda do |args, options|
-      type_stats = {}
-      status_types(options).each do |type|
-        codified = @environment.codified_resources(type)
-        uncodified = @environment.uncodified_resources(type)
-        type_stats[type] = calculate_type_status(codified, uncodified)
-        status_type_rows(type, codified, uncodified, type_stats[type]) if @verbose
-      end
-
+      type_stats = type_stats(options)
       status = calculate_status(type_stats)
-      puts Terminal::Table.new({ rows: status_rows(status) }) if @verbose
+      status = report_json(type_stats, status)
+      status[:resources] = only_codified(status) unless @verbose
       puts JSON.pretty_generate(status)
     end
   end

data/lib/geoengineer/cli/terraform_commands.rb

@@ -15,10 +15,15 @@ module GeoCLI::TerraformCommands
     }
   end
 
+  def terraform_parallelism
+    Parallel.processor_count * 3 # Determined through trial/error
+  end
+
   def terraform_plan
     plan_commands = [
       "cd #{@tmpdir}",
-      "terraform plan -state=#{@terraform_state_file} -out=#{@plan_file} #{@no_color}"
+      "terraform plan -parallelism=#{terraform_parallelism}" \
+      " -state=#{@terraform_state_file} -out=#{@plan_file} #{@no_color}"
     ]
     shell_exec(plan_commands.join(" && "), true)
   end
@@ -26,7 +31,8 @@ module GeoCLI::TerraformCommands
   def terraform_apply
     apply_commands = [
       "cd #{@tmpdir}",
-      "terraform apply -state=#{@terraform_state_file} #{@plan_file} #{@no_color}"
+      "terraform apply -parallelism=#{terraform_parallelism}" \
+      " -state=#{@terraform_state_file} #{@plan_file} #{@no_color}"
     ]
     shell_exec(apply_commands.join(" && "), true)
   end

data/lib/geoengineer/environment.rb

@@ -1,3 +1,4 @@
+require 'parallel'
 
 ########################################################################
 # An Environment is a group of projects, resources and attributes,
@@ -8,6 +9,7 @@
 ########################################################################
 class GeoEngineer::Environment
   include HasAttributes
+  include HasSubResources
   include HasResources
   include HasProjects
   include HasTemplates
@@ -157,7 +159,9 @@ class GeoEngineer::Environment
   def to_terraform_state
     reses = all_resources.select(&:_terraform_id) # _terraform_id must not be nil
 
-    reses = reses.map { |r| { "#{r.type}.#{r.id}" => r.to_terraform_state() } }.reduce({}, :merge)
+    reses = Parallel.map(reses, { in_threads: Parallel.processor_count }) do |r|
+      { "#{r.type}.#{r.id}" => r.to_terraform_state() }
+    end.reduce({}, :merge)
 
     {
       version: 1,

data/lib/geoengineer/resource.rb

@@ -17,7 +17,7 @@ class GeoEngineer::Resource
 
   attr_reader :type, :id
 
-  before :validation, :merge_project_tags
+  before :validation, :merge_parent_tags
 
   validate -> { validate_required_attributes([:_geo_id]) }
 
@@ -113,7 +113,6 @@ class GeoEngineer::Resource
     raw = File.open(path, "rb").read
     interpolated = ERB.new(raw).result(binding_obj)
     escaped = interpolated.gsub("$", "$$")
-
     # normalize JSON to prevent terraform from e.g. newlines as legitimate changes
     normalized = _normalize_json(escaped)
 
@@ -121,8 +120,7 @@ class GeoEngineer::Resource
   end
 
   def _normalize_json(json)
-    h = JSON.parse(json)
-    h.to_json
+    JSON.parse(json).to_json
   end
 
   # REMOTE METHODS
@@ -137,11 +135,9 @@ class GeoEngineer::Resource
     return GeoEngineer::Resource.build(remote_resource_params) if find_remote_as_individual?
 
     matches = matched_remote_resource
+    throw "ERROR:\"#{type}.#{id}\" has #{matches.length} remote resources" if matches.length > 1
 
-    return matches.first if matches.length == 1
-    return nil if matches.empty?
-
-    throw "ERROR:\"#{self.type}.#{self.id}\" has #{matches.length} remote resources"
+    matches.first
   end
 
   # By default, remote resources are bulk-retrieved. In order to fetch a remote resource as an
@@ -159,27 +155,47 @@ class GeoEngineer::Resource
   end
 
   def matched_remote_resource
-    aws_resources = self.class.fetch_remote_resources()
-    aws_resources.select { |r| r._geo_id == self._geo_id }
+    self.class.fetch_remote_resources.select { |r| r._geo_id == _geo_id }
   end
 
   def self.fetch_remote_resources
     return @_rr_cache if @_rr_cache
-    resource_hashes = _fetch_remote_resources()
-    @_rr_cache = resource_hashes.map { |res_hash| GeoEngineer::Resource.build(res_hash) }
+    @_rr_cache = _fetch_remote_resources
+                 .reject { |resource| _ignore_remote_resource?(resource) }
+                 .map { |resource| GeoEngineer::Resource.build(resource) }
   end
 
   # This method must be implemented for each resource type
   # it must return a list of hashes with at least the key
   def self._fetch_remote_resources
-    throw "NOT IMPLEMENTED ERROR for #{self.name}"
+    throw "NOT IMPLEMENTED ERROR for #{name}"
   end
 
-  def self.build(resource_hash)
-    GeoEngineer::Resource.new(self.type_from_class_name, resource_hash['_geo_id']) {
-      resource_hash.each do |k, v|
-        self[k] = v
+  # This method allows you to specify certain remote resources that for whatever reason,
+  # cannot or should not be codified. It expects a list of `_geo_ids`, and can be overriden
+  # in child classes.
+  def self._resources_to_ignore
+    []
+  end
+
+  def self._ignore_remote_resource?(resource)
+    _resources_to_ignore.include?(_deep_symbolize_keys(resource)[:_geo_id])
+  end
+
+  def self._deep_symbolize_keys(obj)
+    case obj
+    when Hash then
+      obj.each_with_object({}) do |(key, value), hash|
+        hash[key.to_sym] = _deep_symbolize_keys(value)
       end
+    when Array then obj.map { |value| _deep_symbolize_keys(value) }
+    else obj
+    end
+  end
+
+  def self.build(resource_hash)
+    GeoEngineer::Resource.new(type_from_class_name, resource_hash['_geo_id']) {
+      resource_hash.each { |k, v| self[k] = v }
     }
   end
 
@@ -196,8 +212,7 @@ class GeoEngineer::Resource
   def short_id
     si = id.to_s.tr('-', "_")
     si = si.gsub(project.full_id_name, '') if project
-    si = si.gsub('__', '_').gsub(/^_|_$/, '')
-    si
+    si.gsub('__', '_').gsub(/^_|_$/, '')
   end
 
   def short_name
@@ -216,19 +231,26 @@ class GeoEngineer::Resource
     tags {} unless tags
   end
 
-  def merge_project_tags
-    return unless self.project && self.project.tags && self.support_tags?
+  def merge_parent_tags
+    return unless support_tags?
 
+    %i(project environment).each do |source|
+      parent = send(source)
+      next unless parent
+      next unless parent.methods.include?(:attributes)
+      next unless parent&.tags
+      merge_tags(source)
+    end
+  end
+
+  def merge_tags(source)
     setup_tags_if_needed
 
-    self
-      .project
+    send(source)
       .all_tags
       .map(&:attributes)
       .reduce({}, :merge)
       .each { |key, value| tags.attributes[key] ||= value }
-
-    tags
   end
 
   # VALIDATION METHODS
@@ -237,17 +259,15 @@ class GeoEngineer::Resource
   end
 
   def validate_required_subresource(subresource)
-    "Subresource '#{subresource}'' required #{for_resource}" if self.send(subresource.to_sym).nil?
+    "Subresource '#{subresource}'' required #{for_resource}" if send(subresource.to_sym).nil?
   end
 
   def validate_subresource_required_attributes(subresource, keys)
-    errs = []
-    self.send("all_#{subresource}".to_sym).each do |sr|
-      keys.each do |key|
-        errs << "#{key} attribute on subresource #{subresource} nil #{for_resource}" if sr[key].nil?
+    send("all_#{subresource}".to_sym).map do |sr|
+      keys.map do |key|
+        "#{key} attribute on subresource #{subresource} nil #{for_resource}" if sr[key].nil?
       end
-    end
-    errs
+    end.flatten.compact
   end
 
   def validate_has_tag(tag)
@@ -260,10 +280,9 @@ class GeoEngineer::Resource
   # CLASS METHODS
   def self.type_from_class_name
     # from http://stackoverflow.com/questions/1509915/converting-camel-case-to-underscore-case-in-ruby
-    self.name.split('::').last
+    name.split('::').last
         .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
         .gsub(/([a-z\d])([A-Z])/, '\1_\2')
-        .tr("-", "_")
-        .downcase
+        .tr("-", "_").downcase
   end
 end

data/lib/geoengineer/resources/aws_cloudwatch_metric_alarm.rb

@@ -0,0 +1,48 @@
+########################################################################
+# AwsCloudwatchMetricAlarm is the +aws_cloudwatch_metric_alarm+ terrform resource,
+#
+# {https://www.terraform.io/docs/providers/aws/r/aws_cloudwatch_metric_alarm.html Terraform Docs}
+########################################################################
+class GeoEngineer::Resources::AwsCloudwatchMetricAlarm < GeoEngineer::Resource
+  validate -> {
+    validate_required_attributes([
+                                   :alarm_name,
+                                   :comparison_operator,
+                                   :evaluation_periods,
+                                   :metric_name,
+                                   :namespace,
+                                   :period,
+                                   :threshold
+                                 ])
+  }
+
+  after :initialize, -> { _terraform_id -> { NullObject.maybe(remote_resource)._terraform_id } }
+  after :initialize, -> { _geo_id       -> { alarm_name } }
+
+  def support_tags?
+    false
+  end
+
+  def self._fetch_remote_resources
+    _get_all_alarms.map { |alarm|
+      {
+        _terraform_id: alarm[:alarm_name],
+        _geo_id: alarm[:alarm_name],
+        alarm_name: alarm[:alarm_name]
+      }
+    }
+  end
+
+  def self._get_all_alarms
+    alarm_page = AwsClients.cloudwatch.describe_alarms({ max_records: 100 })
+    alarms = alarm_page.metric_alarms.map(&:to_h)
+    while alarm_page.next_token
+      alarm_page = AwsClients.cloudwatch.describe_alarms({
+                                                           max_records: 100,
+                                                           next_token: alarm_page.next_token
+                                                         })
+      alarms.concat alarm_page.metric_alarms.map(&:to_h)
+    end
+    alarms
+  end
+end