marty 1.1.5 → 1.1.6

data/app/models/diagnostic/aws/ec2_instance.rb

@@ -0,0 +1,100 @@
+class Diagnostic::Aws::Ec2Instance
+  attr_reader :id, :doc, :role, :creds, :version, :host, :tag, :nodes
+
+  # aws reserved host used to get instance meta-data
+  META_DATA_HOST = '169.254.169.254'
+
+  def self.is_aws?
+    response = get("http://#{META_DATA_HOST}") rescue nil
+    response.present?
+  end
+
+  def initialize
+    @id      = get_instance_id
+    @doc     = get_document
+    @role    = get_role
+    @creds   = get_credentials
+    @host    = "ec2.#{@doc['region']}.amazonaws.com"
+    @version = '2016-11-15'
+    @tag     = get_tag
+    @nodes   = get_private_ips
+  end
+
+  private
+  def self.get url
+    uri = URI.parse(url)
+    request = Net::HTTP.new(uri.host, uri.port)
+    request.read_timeout = request.open_timeout = ENV['DIAG_TIMEOUT'] || 5
+    request.start {|http|
+      http.get(uri.to_s)
+    }.body
+  end
+
+  def query_meta_data query
+    self.class.get("http://#{META_DATA_HOST}/latest/meta-data/#{query}/")
+  end
+
+  def query_dynamic query
+    self.class.get("http://#{META_DATA_HOST}/latest/dynamic/#{query}/")
+  end
+
+  def get_instance_id
+    query_meta_data('instance-id').to_s
+  end
+
+  def get_role
+    query_meta_data('iam/security-credentials').to_s
+  end
+
+  def get_credentials
+    JSON.parse(query_meta_data("iam/security-credentials/#{@role}"))
+  end
+
+  def get_document
+    JSON.parse(query_dynamic('instance-identity/document'))
+  end
+
+  def ec2_request action, params = {}
+    default = {
+      'Action' => action,
+      'Version' => @version
+    }
+
+    url = "https://#{@host}/?" +
+          (default + params).map{|a, v| "#{a}=#{v}"}.join('&')
+
+    sig = Aws::Sigv4::Signer.new(service:           'ec2',
+                                 region:            @doc['region'],
+                                 access_key_id:     @creds['AccessKeyId'],
+                                 secret_access_key: @creds['SecretAccessKey'],
+                                 session_token:     @creds['Token'])
+    signed_url = sig.presign_url(http_method:'GET', url: url)
+
+    http = Net::HTTP.new(@host, 443)
+    http.use_ssl = true
+    Hash.from_xml(Net::HTTP.get(signed_url))["#{action}Response"]
+  end
+
+  def get_tag
+    params = {'Filter.1.Name'    => 'resource-id',
+              'Filter.1.Value.1' => get_instance_id,
+              'Filter.2.Name'    => 'key',
+              'Filter.2.Value.1' => 'Name'}
+    ec2_request('DescribeTags', params)['tagSet']['item']['value']
+  end
+
+  def get_instances
+    params = {'Filter.1.Name'    => 'tag-value',
+              'Filter.1.Value.1' => @tag}
+    ec2_request('DescribeInstances', params)
+  end
+
+  def get_private_ips
+    get_instances['reservationSet']['item'].map{
+      |i|
+      item = i['instancesSet']['item']
+      item.is_a?(Array) ? item.map{|i| i['privateIpAddress']} :
+        item['privateIpAddress']
+    }.flatten
+  end
+end

data/app/models/diagnostic/base.rb

@@ -0,0 +1,69 @@
+class Diagnostic::Base < Diagnostic::Request
+  extend Diagnostic::Packer
+  include ActionView::Helpers::TextHelper
+
+  # all diagnostics have `aggregatable` set to true.
+  # aggregatable indicates to the reporting mechanism that a diagnostic
+  # should be aggregated  as these types of diagnostics are
+  # aggregated differently (or not at all).
+  class_attribute :aggregatable, :status_only
+  self.aggregatable = true
+  self.status_only = false
+
+  @@read_only = Marty::Util.db_in_recovery?
+  @@template  = ActionController::Base.new.lookup_context.
+                find_template("marty/diagnostic/diag").identifier
+
+  def self.generate
+    raise "generate has not been defined for #{name}"
+  end
+
+  def self.fatal?
+    name == 'Diagnostic::Fatal'
+  end
+
+  def self.process_status_only infos
+    return infos unless status_only
+    infos.map{|info| info.map{|test, result| [test, result['status']]}.to_h}
+  end
+
+  def self.get_difference data
+    values = process_status_only(data.values)
+    Marty::DataExporter.hash_array_merge(values, true).map{
+      |test, values|
+      test if values.uniq.count > 1
+    }.compact
+  end
+
+  def self.apply_consistency data
+    diff = get_difference(data)
+    data.each_with_object({}){
+      |(node, diagnostic), new_data|
+      new_data[node] = diagnostic.each_with_object({}){
+        |(test, info), new_diagnostic|
+        new_diagnostic[test] = info + {'consistent' => !diff.include?(test)}
+      }
+    }
+  end
+
+  def self.consistent? data
+    process_status_only(data.values).uniq.count == 1
+  end
+
+  def self.display data
+    consistent = consistent?(data)
+    success = consistent && !fatal?
+    ERB.new(File.open(@@template).read).result(binding)
+  end
+
+  def self.display_info_css info
+    return 'inconsistent' if info.nil? || (info['status'] &&
+                                           info['consistent'] == false)
+    return 'error' unless info['status']
+    'passed'
+  end
+
+  def self.display_info_description info
+    new.simple_format(info ? info['description'] : 'N/A')
+  end
+end

data/app/models/diagnostic/base_collection.rb

@@ -0,0 +1,10 @@
+# used to group separate diagnostics into one diagnostic
+class Diagnostic::Collection < Diagnostic::ByStatus
+  class_attribute :diagnostics
+  self.diagnostics = []
+
+  def self.generate
+    raise 'No diagnostics assigned to collection.' if diagnostics.empty?
+    diagnostics.map{|diagnostic| diagnostic.generate}.reduce(:deep_merge)
+  end
+end

data/app/models/diagnostic/collection.rb

@@ -0,0 +1,10 @@
+class Diagnostic::Collection < Diagnostic::Base
+  class_attribute :diagnostics
+  self.diagnostics = []
+  self.status_only = true
+
+  def self.generate
+    raise 'No diagnostics assigned to collection.' if diagnostics.empty?
+    diagnostics.map{|d| d.generate}.reduce(:deep_merge)
+  end
+end

data/app/models/diagnostic/delayed_job.rb

@@ -0,0 +1,48 @@
+# DelayedJob is a unique diagnostic that creates a series of delayed jobs
+# in the hopes that enough nodes will touch these jobs to determine
+# if delayed job workers are running the latest application version
+#
+# `DELAYED_VER` environment variable should be set in the
+# delayed jobs initializer.
+#
+class Diagnostic::DelayedJob < Diagnostic::Base
+  self.aggregatable = false
+
+  def self.generate
+    raise 'DelayedJob cannot be called with local scope.' if scope == 'local'
+
+    raise 'DELAYED_VER environment variable has not been initialized.' if
+      ENV['DELAYED_VER'].nil?
+
+    total_workers = delayed_worker_count
+
+    raise 'No delayed jobs are running.' if total_workers.zero?
+
+    # we will only iterate by half of the total delayed workers to avoid
+    # excess use of delayed job time
+    total_workers = (total_workers/2).zero? ? 1 : total_workers/2
+
+    d_engine = Marty::ScriptSet.new.get_engine("Diagnostics")
+    res = d_engine.
+            evaluate('VersionDelay', 'result', {'count' => total_workers-1})
+
+    # merge results, remove duplicates, and construct "aggregate" object
+    res.each_with_object({}){
+      |r, hash|
+      hash[r[0]] ||= []
+      hash[r[0]] << r[1]
+    }.map {
+      |node, result|
+
+      versions = result.uniq
+      status = versions.count == 1 && versions[0] == ENV['DELAYED_VER']
+
+      {node => {'Version' => create_info(versions.join("\n"), status)}}
+    }.reduce(:deep_merge)
+  end
+
+  def self.delayed_worker_count
+    Diagnostic::Node.get_postgres_connections[Diagnostic::Database.db_name].
+      count{|conn| conn['application_name'].include?('delayed_job')}
+  end
+end

data/app/models/diagnostic/env.rb

@@ -0,0 +1,35 @@
+class Diagnostic::Env < Diagnostic::Base
+  def self.environment_variables filter=''
+    env = ENV.clone
+
+    # obfuscate SECRET_KEY_BASE for comparison
+    env['SECRET_KEY_BASE'] = env['SECRET_KEY_BASE'][0,4] if
+      env['SECRET_KEY_BASE']
+
+    # remove SCRIPT_URI, SCRIPT_URL as calling node differs
+    ['SCRIPT_URI', 'SCRIPT_URL'].each{|k| env.delete(k)}
+
+    to_block = ['PASSWORD', 'DEBUG']
+    env.sort.each_with_object({}){|(k,v),h|
+      h[k] = v if to_block.all?{|b| !k.include?(b)} && k.include?(filter)}
+  end
+
+  def self.generate
+    pack do
+      environment_variables
+    end
+  end
+
+  # overwritten to only return inconsitent data
+  def self.apply_consistency data
+    diff = get_difference(data)
+    data.each_with_object({}){
+      |(node, diagnostic), new_data|
+      new_data[node] = diagnostic.each_with_object({}){
+        |(test, info), new_diagnostic|
+        new_diagnostic[test] = info + {'consistent' => false} if
+          diff.include?(test)
+      }
+    }
+  end
+end

data/app/models/diagnostic/environment.rb

@@ -0,0 +1,35 @@
+class Diagnostic::Environment < Diagnostic::Base
+  def self.generate
+    pack do
+      rbv = "#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL} (#{RUBY_PLATFORM})"
+      {
+        'Environment'             => Rails.env,
+        'Rails'                   => Rails.version,
+        'Netzke Core'             => Netzke::Core::VERSION,
+        'Netzke Basepack'         => Netzke::Basepack::VERSION,
+        'Ruby'                    => rbv,
+        'RubyGems'                => Gem::VERSION,
+        'Database Adapter'        => Diagnostic::Database.db_adapter_name,
+        'Database Server'         => Diagnostic::Database.db_server_name,
+        'Database Version'        => db_version,
+        'Database Schema Version' => db_schema
+      }
+    end
+  end
+
+  def self.db_version
+    begin
+      Diagnostic::Database.db_version
+    rescue => e
+      error(e.message)
+    end
+  end
+
+  def self.db_schema
+    begin
+      Diagnostic::Database.db_schema
+    rescue => e
+      error(e.message)
+    end
+  end
+end

data/app/models/diagnostic/fatal.rb

@@ -0,0 +1,12 @@
+class Diagnostic::Fatal < Diagnostic::Base
+  def self.display_alert_message
+    '<h3 class="error">Something went wrong.</br>'\
+    'Consistency is checked between remaining nodes if applicable.</h3>'
+  end
+
+  def self.message msg, opts = {}
+    node = opts[:node] || Diagnostic::Node.my_ip
+    type = opts[:type] || 'RuntimeError'
+    {name => {node => {type => error(msg)}}}
+  end
+end

data/app/models/diagnostic/helper.rb

@@ -0,0 +1,11 @@
+class Diagnostic::Helper
+  include Delorean::Model
+
+  delorean_fn :my_ip, sig:0 do
+    Diagnostic::Node.my_ip
+  end
+
+  delorean_fn :git, sig:0 do
+    [my_ip, ENV['DELAYED_VER']]
+  end
+end

data/app/models/diagnostic/nodes.rb

@@ -0,0 +1,18 @@
+class Diagnostic::Nodes < Diagnostic::Base
+  def self.generate
+    pack do
+      begin
+        a_nodes = Diagnostic::Aws::Ec2Instance.new.nodes.sort if
+          Diagnostic::Aws::Ec2Instance.is_aws?
+      rescue => e
+        a_nodes = [e.message]
+      end
+      pg_nodes = Diagnostic::Node.get_nodes.sort
+      a_nodes.nil? || pg_nodes == a_nodes ? pg_nodes.join("\n") :
+        error("There is a discrepancy between nodes connected to "\
+              "Postgres and those discovered through AWS EC2.\n"\
+              "Postgres: \n#{pg_nodes.join("\n")}\n"\
+              "AWS: \n#{a_nodes.join("\n")}")
+    end
+  end
+end

data/app/models/diagnostic/reporter.rb

@@ -0,0 +1,108 @@
+class Diagnostic::Reporter < Diagnostic::Request
+  class_attribute :reports, :diagnostics
+
+  self.reports = {}
+  self.diagnostics = []
+
+  def self.run request
+    self.request = request
+
+    ops = op.split(/,\s*/).uniq - [unresolve_diagnostic(self)]
+    reps = ops.select{|o| reports.keys.include?(o)}
+
+    self.diagnostics = ((ops - reps) + reps.map{|r| reports[r]}.flatten).uniq.
+                         map{|d| resolve_diagnostic(d)}
+
+    self.scope == 'local' ? generate : aggregate
+  end
+
+  private
+  def self.resolve_diagnostic name
+    return name.constantize unless name.slice('Diagnostic::').nil?
+    ('Diagnostic::' + name.downcase.camelize).constantize
+  end
+
+  def self.unresolve_diagnostic klass
+    klass.name.demodulize.underscore
+  end
+
+  def self.generate
+    diagnostics.each_with_object({}){
+      |d, h|
+      begin
+        h[d.name] = d.generate
+      rescue => e
+        h.deep_merge!(Diagnostic::Fatal.message(e.message, type: d.name))
+      end
+    }
+  end
+
+  def self.aggregate
+    data = consistency([generate, get_remote_diagnostics].reduce(:deep_merge))
+    {'data' => data, 'errors' => errors(data)}
+  end
+
+  def self.consistency data
+    data.each_with_object({}){
+      |(klass, result), h|
+      h[klass] = resolve_diagnostic(klass).apply_consistency(result)
+    }
+  end
+
+  def self.errors data
+    data.each_with_object({}){
+      |(klass, result), new_data|
+      new_data[klass] = result.each_with_object({}){
+        |(node, diagnostic), new_result|
+        new_result[node] = diagnostic.each_with_object({}){
+          |(test, info), new_diagnostic|
+          new_diagnostic[test] = info unless
+            info['status'] && (scope == 'local' || info['consistent'])
+        }
+        new_result.delete(node) if new_result[node].empty?
+      }
+      new_data.delete(klass) if new_data[klass].empty?
+    }
+  end
+
+  def self.displays result
+    result.map{|d, r| resolve_diagnostic(d).display(r)}.sum
+  end
+
+  def self.get_remote_diagnostics
+    ops = diagnostics.map{|d| unresolve_diagnostic(d) if d.aggregatable}.compact
+    return {} if ops.empty?
+
+    nodes = Diagnostic::Node.get_nodes - [Diagnostic::Node.my_ip]
+    remote = nodes.sort.map do |n|
+      Thread.new do
+        uri = Addressable::URI.new(host: n, port: request.port)
+        uri.scheme = ssl? ? 'https' : 'http'
+        uri.path = '/marty/diag.json'
+        uri.query_values = {
+          scope: 'local',
+          op: ops.join(','),
+        }
+        req = Net::HTTP.new(uri.host, uri.port)
+        req.use_ssl = ssl?
+        req.read_timeout = req.open_timeout = ENV['DIAG_TIMEOUT'] || 10
+        req.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+        begin
+          response = req.start {|http| http.get(uri.to_s)}
+          next JSON.parse(response.body) if response.code == "200"
+
+          Diagnostic::Fatal.message(response.body,
+                                    type: response.message,
+                                    node: uri.host)
+        rescue => e
+          Diagnostic::Fatal.message(e.message,
+                                    type: e.class,
+                                    node: uri.host)
+        end
+      end
+    end
+
+    remote.empty? ? {} : remote.map(&:join).map(&:value).reduce(:deep_merge)
+  end
+end