marty 2.3.15 → 2.4.0

data/app/models/marty/api_auth.rb

@@ -1,11 +1,11 @@
 class Marty::ApiAuth < Marty::Base
   has_mcfly
 
+  belongs_to :entity, polymorphic: true, optional: true
+
   KEY_SIZE = 19
 
-  def self.generate_key
-    SecureRandom.hex(KEY_SIZE)
-  end
+  validates_presence_of :app_name, :api_key, :script_name
 
   class ApiAuthValidator < ActiveModel::Validator
     def validate(api)
@@ -17,24 +17,96 @@ class Marty::ApiAuth < Marty::Base
     end
   end
 
+  validates_with ApiAuthValidator
+
+  mcfly_validates_uniqueness_of :api_key, scope: [:script_name]
+  validates_uniqueness_of :app_name, scope: [:script_name,
+                                             :obsoleted_dt]
+
+
   before_validation do
     self.api_key = Marty::ApiAuth.generate_key if
       self.api_key.nil? || self.api_key.length == 0
   end
 
-  validates_presence_of :app_name, :api_key, :script_name
+  before_save do
+    return unless changed.include?(:entity_id) && !parameters['aws_api_key']
 
-  validates_with ApiAuthValidator
+    msg = 'API Auth must be associated with an AWS API KEY before '\
+          'it can be associated with an entity'
 
-  mcfly_validates_uniqueness_of :api_key, scope: [:script_name]
-  validates_uniqueness_of :app_name, scope: [:script_name,
-                                             :obsoleted_dt]
+    errors.add(:base, msg)
+  end
+
+  before_destroy do
+    next unless aws = parameters['aws_api_key']
+    begin
+      client = Marty::Aws::Apigateway.new
+      resp   = client.delete_usage_plan_key(aws['api_usage_plan_id'],
+                                            aws['aid'])
+      client.delete_api_key(aws['aid']) if resp
+    rescue => e
+      Marty::Logger.log('api_test', 'error', e.message)
+      throw :abort unless e.message.include?('Invalid API Key')
+    end
+  end
+
+  def self.generate_key
+    SecureRandom.hex(KEY_SIZE)
+  end
+
+  def create_aws_api_key api_id, api_usage_plan_id
+    client = Marty::Aws::Apigateway.new
+    app_id = Marty::Config['AWS_APP_IDENTIFIER'] || 'marty'
+    name   = "#{app_id}-#{api_id}-#{api_key[0..3]}"
+
+    key = nil
+    begin
+      key = client.create_api_key(name, 'marty_api_key', api_key)
+    rescue => e
+      #Marty::Logger.log('api_test', 'error', e.message)
+    end
+
+    upkey = nil
+    begin
+    upkey = key &&
+            client.create_usage_plan_key(api_usage_plan_id, key.id)
+    rescue => e
+      #Marty::Logger.log('api_test', 'error', e.message)
+      # remove api key we created
+      client.delete_api_key(key.id)
+    end
+
+    raise "Unable to create AWS API Key" unless key && upkey
+
+    parameters['aws_api_key'] = {
+      'aid'               => key.id,
+      'api_usage_plan_id' => api_usage_plan_id,
+      'api_id'            => api_id,
+    }
+
+    save!
+  end
+
+  def move_aws_key usage_plan_id
+    return unless aws = parameters['aws_api_key']
+    return if aws['api_usage_plan_id'] == usage_plan_id
+
+    begin
+      client = Marty::Aws::Apigateway.new
+      resp   = client.delete_usage_plan_key(aws['api_usage_plan_id'],
+                                            aws['aid'])
+    rescue => e
+      # on fail recreate usage plan key
+      Marty::Logger.log('api', 'api_test', aws)
+      client.create_usage_plan_key(aws['api_usage_plan_id'], aws['aid']) if
+        client
+      return
+    else
+      client.create_usage_plan_key(usage_plan_id, aws['aid']) if resp
+    end
 
-  def self.authorized?(script_name, api_key)
-    is_secured = where(script_name: script_name,
-                       obsoleted_dt: 'infinity').exists?
-    !is_secured || where(api_key: api_key,
-                         script_name: script_name,
-                         obsoleted_dt: 'infinity').pluck(:app_name).first
+    parameters['aws_api_key'] += {'api_usage_plan_id' => usage_plan_id}
+    save!
   end
 end

data/app/models/marty/api_config.rb

@@ -3,13 +3,18 @@ class Marty::ApiConfig < Marty::Base
 
   def self.lookup(script, node, attr)
     res = where(["script = ? AND (node IS NULL OR node = ?) "\
-           "AND (attr IS NULL OR attr = ?)",
-           script, node, attr]).
-           order('node nulls last, attr nulls last').
-           pluck(:logged, :input_validated, :output_validated, :strict_validate,
-                 :id)
-    res.first
+                 "AND (attr IS NULL OR attr = ?)",
+                 script, node, attr]).
+            order('node nulls last, attr nulls last').first
+
+    res && res.as_json.except('id',
+                              'created_at',
+                              'updated_at',
+                              'script',
+                              'node',
+                              'attr').symbolize_keys
   end
+
   def self.multi_lookup(script, node, attrs)
     (attrs.nil? ? [nil] : attrs).
       map { |attr| lookup(script, node, attr).try{|x| x.unshift(attr) }}

data/app/models/marty/delorean_rule.rb

@@ -152,9 +152,10 @@ class Marty::DeloreanRule < Marty::BaseRule
         estack_full = resh.delete(:err_stack)
         estack = estack_full && {
           err_stack: estack_full.select{ |l| l.starts_with?('DELOREAN')}} || {}
-        detail = { input: params, dgparams: dgparams} + resh + estack
         Marty::Logger.info("Rule Log #{ruleh['name']}",
-                           Marty::Util.scrub_obj(detail))
+                           { input: params,
+                             dgparams: dgparams } + resh + estack
+                          )
       end
     end
   end

data/config/routes.rb

@@ -6,5 +6,5 @@ Marty::Engine.routes.draw do
   match via: [:get, :post], "rpc/evaluate(.:format)" => "rpc", as: :rpc
   match via: [:get, :post], "report(.:format)" => "report#index", as: :report
   get  'job/download' => 'job', as: :job
-  get  'diag', to: 'diagnostic/#op'
+  get  'diag',        to: 'diagnostic/#op'
 end

data/db/migrate/501_add_api_class_to_marty_api_config.rb

@@ -0,0 +1,6 @@
+class AddApiClassToMartyApiConfig < ActiveRecord::Migration[5.1]
+  def change
+    table = :marty_api_configs
+    add_column table, :api_class, :string, default: 'Marty::Api::Base'
+  end
+end

data/db/migrate/502_add_parameters_to_marty_api_auth.rb

@@ -0,0 +1,5 @@
+class AddParametersToMartyApiAuth < ActiveRecord::Migration[5.1]
+  def change
+    add_column :marty_api_auths, :parameters, :jsonb, default: {}
+  end
+end

data/lib/marty/util.rb

@@ -132,19 +132,4 @@ module Marty::Util
     URI.encode("#{Marty::Util.marty_path}/report?data=#{data}"\
                "&reptitle=#{title}&format=#{format}")
   end
-
-  def self.scrub_obj(obj)
-    trav = lambda {|o|
-           if o.is_a?(Hash)
-             return o.each_with_object({}) {|(k, v), h| h[k] = trav.call(v)}
-           elsif o.is_a?(Array)
-             return o.map {|v| trav.call(v)}
-           elsif o.to_s.length > 10000
-             o.class.to_s
-           else
-             o
-           end
-    }
-    trav.call(obj)
-  end
 end

data/lib/marty/version.rb

@@ -1,3 +1,3 @@
 module Marty
-  VERSION = "2.3.15"
+  VERSION = "2.4.0"
 end

data/other/marty/api/base.rb

@@ -0,0 +1,207 @@
+class Marty::Api::Base
+  mattr_accessor :class_list
+  @@class_list ||= [name].to_set
+
+  def self.inherited(klass)
+    @@class_list << klass.to_s
+    super
+  end
+
+  def self.respond_to controller
+    result = yield
+    controller.respond_to do |format|
+      format.json { controller.send_data result.to_json }
+      format.csv  {
+        # SEMI-HACKY: strip outer list if there's only one element.
+        result = result[0] if result.is_a?(Array) && result.length==1
+        controller.send_data Marty::DataExporter.to_csv(result)
+      }
+    end
+  end
+
+  # api handles
+  def self.process_params params
+    params
+  end
+
+  def self.before_evaluate api_params
+  end
+
+  def self.after_evaluate api_params, result
+  end
+
+  def self.is_authorized? params
+    is_secured = Marty::ApiAuth.where(
+      script_name: params[:script],
+      obsoleted_dt: 'infinity'
+    ).exists?
+
+    !is_secured || Marty::ApiAuth.where(
+      api_key:       params[:api_key],
+      script_name:   params[:script],
+      obsoleted_dt: 'infinity'
+    ).pluck(:app_name).first
+  end
+
+  def self.evaluate params, request, config
+    # validate input schema
+    if config[:input_validated]
+      begin
+        schema = SchemaValidator::get_schema(params)
+      rescue => e
+        return {error: e.message}
+      end
+
+      begin
+        res = SchemaValidator::validate_schema(schema, params[:params])
+      rescue NameError
+        return {error: "Unrecognized PgEnum for attribute #{params[:attr]}"}
+      rescue => e
+        return {error: "#{params[:attr]}: #{e.message}"}
+      end
+
+      schema_errors = SchemaValidator::get_errors(res) unless res.empty?
+      return {error: "Error(s) validating: #{schema_errors}"} if
+        schema_errors
+    end
+
+    # get script engine
+    begin
+      engine = Marty::ScriptSet.new(params[:tag]).get_engine(params[:script])
+    rescue => e
+      error = "Can't get engine: #{params[:script] || 'nil'} with tag: " +
+                "#{params[:tag] || 'nil'}; message: #{e.message}"
+      Marty::Logger.info error
+      return {error: error}
+    end
+
+    retval = nil
+
+    # evaluate script
+    begin
+      if params[:background]
+        res = engine.background_eval(params[:node],
+                                     params[:params],
+                                     params[:attr])
+
+        return retval = {"job_id" => res.__promise__.id}
+      end
+
+      res = engine.evaluate(params[:node],
+                            params[:attr],
+                            params[:params])
+
+      # validate output schema
+      if config[:output_validated] && !(res.is_a?(Hash) && res['error'])
+        begin
+          output_schema_params = params + {attr: params[:attr] + '_'}
+          schema = SchemaValidator::get_schema(output_schema_params)
+        rescue => e
+          return {error: e.message}
+        end
+
+        begin
+          schema_errors = SchemaValidator::validate_schema(schema, res)
+        rescue NameError
+          return {error: "Unrecognized PgEnum for attribute #{attr}"}
+        rescue => e
+          return {error: "#{attr}: #{e.message}"}
+        end
+
+        if schema_errors.present?
+          errors = schema_errors.map{|e| e[:message]}
+
+          Marty::Logger.error(
+            "API #{params[:script]}:#{params[:node]}.#{params[:attr]}",
+            {error: errors, data: res}
+          )
+
+          msg = "Error(s) validating: #{errors}"
+          res = config[:strict_validate] ? {error: msg ,data: res} : res
+        end
+      end
+
+      # if attr is an array, return result as an array
+      return retval = params[:return_array] ? [res] : res
+
+    rescue => e
+      msg = Delorean::Engine.grok_runtime_exception(e).symbolize_keys
+      Marty::Logger.info "Evaluation error: #{msg}"
+      return retval = msg
+    ensure
+      error = Hash === retval ? retval[:error] : nil
+    end
+  end
+
+  def self.log result, params, request
+    ret_arr = params[:return_array]
+    Marty::Log.write_log('api',
+                         params.values_at(:script, :node, :attr).join(' - '),
+                         {script:     params[:script],
+                          node:       params[:node],
+                          attrs:      ret_arr ? [params[:attr]] : params[:attr],
+                          input:      params[:params],
+                          output:     (result.is_a?(Hash) &&
+                                       result.include?('error')) ? nil : result,
+                          start_time: params[:start_time],
+                          end_time:   Time.zone.now,
+                          error:      (result.is_a?(Hash) &&
+                                       result.include?('error')) ? result : nil,
+                          remote_ip:  request.remote_ip,
+                          auth_name:  params[:auth]
+                         })
+  end
+
+  class SchemaValidator
+    def self.get_schema params
+      begin
+        Marty::ScriptSet.new(params[:tag]).get_engine(params[:script]+'Schemas').
+          evaluate(params[:node], params[:attr], {})
+      rescue => e
+        msg = e.message == 'No such script' ? 'Schema not defined' :
+                'Problem with schema: ' + e.message
+
+        raise "Schema error for #{params[:script]}/#{params[:node]} "\
+              "attrs=#{params[:attr]}: #{msg}"
+      end
+    end
+
+    def self.validate_schema schema, hash
+      JSON::Validator.fully_validate(
+        schema.merge({"\$schema" => Marty::JsonSchema::RAW_URI}),
+        hash,
+        validate_schema:   true,
+        errors_as_objects: true,
+        version:           Marty::JsonSchema::RAW_URI,
+      )
+    end
+
+    def self.massage_message(msg)
+      m = %r|'#/([^']+)' of type ([^ ]+) matched the disallowed schema|.
+            match(msg)
+
+      return msg unless m
+      "disallowed parameter '#{m[1]}' of type #{m[2]} was received"
+    end
+
+    def self._get_errors(errs)
+      if errs.is_a?(Array)
+        errs.map { |err| _get_errors(err) }
+      elsif errs.is_a?(Hash)
+        if !errs.include?(:failed_attribute)
+          errs.map { |k, v| _get_errors(v) }
+        else
+          fa, fragment, message, errors = errs.values_at(:failed_attribute,
+                                                         :fragment,
+                                                         :message, :errors)
+          ((['AllOf','AnyOf','Not'].include?(fa) && fragment =='#/') ?
+             [] : [massage_message(message)]) + _get_errors(errors || {})
+        end
+      end
+    end
+
+    def self.get_errors(errs)
+      _get_errors(errs).flatten
+    end
+  end
+end

data/other/marty/diagnostic/aws/ec2_instance.rb

@@ -1,13 +1,8 @@
-class Marty::Diagnostic::Aws::Ec2Instance
+class Marty::Diagnostic::Aws::Ec2Instance < Marty::Aws::Base
   # aws reserved host used to get instance meta-data
   META_DATA_HOST = '169.254.169.254'
 
-  attr_reader :id,
-              :doc,
-              :role,
-              :creds,
-              :version,
-              :host,
+  attr_reader :host,
               :tag,
               :nodes,
               :instances
@@ -33,76 +28,18 @@ class Marty::Diagnostic::Aws::Ec2Instance
     end
   end
 
-  def self.is_aws?
-    response = get("http://#{META_DATA_HOST}") rescue nil
-    response.present?
-  end
-
   def initialize
-    @id            = get_instance_id
-    @doc           = get_document
-    @role          = get_role
-    @creds         = get_credentials
-    @host          = "ec2.#{@doc['region']}.amazonaws.com"
-    @version       = '2016-11-15'
-    @tag           = get_tag
-    @instances     = InstancesSet.new(get_instances)
-    @nodes         = get_private_ips
-  end
-
-  def self.get url
-    uri = URI.parse(url)
-    request = Net::HTTP.new(uri.host, uri.port)
-    request.read_timeout = request.open_timeout = ENV['DIAG_TIMEOUT'] || 0.25
-    request.start {|http|
-      http.get(uri.to_s)
-    }.body
-  end
-
-  def query_meta_data query
-    self.class.get("http://#{META_DATA_HOST}/latest/meta-data/#{query}/")
-  end
-
-  def query_dynamic query
-    self.class.get("http://#{META_DATA_HOST}/latest/dynamic/#{query}/")
+    super
+    @service   = 'ec2'
+    @tag       = get_tag
+    @instances = InstancesSet.new(get_instances)
+    @nodes     = get_private_ips
   end
 
   private
-  def get_instance_id
-    query_meta_data('instance-id').to_s
-  end
-
-  def get_role
-    query_meta_data('iam/security-credentials').to_s
-  end
-
-  def get_credentials
-    JSON.parse(query_meta_data("iam/security-credentials/#{@role}"))
-  end
-
-  def get_document
-    JSON.parse(query_dynamic('instance-identity/document'))
-  end
-
   def ec2_request action, params = {}
-    default = {
-      'Action' => action,
-      'Version' => @version
-    }
-
-    url = "https://#{@host}/?" +
-          (default + params).map{|a, v| "#{a}=#{v}"}.join('&')
-
-    sig = Aws::Sigv4::Signer.new(service:           'ec2',
-                                 region:            @doc['region'],
-                                 access_key_id:     @creds['AccessKeyId'],
-                                 secret_access_key: @creds['SecretAccessKey'],
-                                 session_token:     @creds['Token'])
-    signed_url = sig.presign_url(http_method:'GET', url: url)
-
-    http = Net::HTTP.new(@host, 443)
-    http.use_ssl = true
-    Hash.from_xml(Net::HTTP.get(signed_url))["#{action}Response"]
+    resp = request({action: action}, params)
+    Hash.from_xml(resp)["#{action}Response"]
   end
 
   def get_tag
@@ -117,9 +54,11 @@ class Marty::Diagnostic::Aws::Ec2Instance
     params = {'Filter.1.Name'    => 'tag-value',
               'Filter.1.Value.1' => @tag}
 
+    resp = ec2_request('DescribeInstances', params)
+
     instances = ensure_resp(
       ['reservationSet', 'item', 'instancesSet', 'item'],
-      ec2_request('DescribeInstances', params)
+      resp
     ).map do |i|
       {
         'id'    => i['instanceId'],
@@ -132,18 +71,4 @@ class Marty::Diagnostic::Aws::Ec2Instance
   def get_private_ips
     @instances.running.map{|i| i['ip']}.compact
   end
-
-  def ensure_resp path, obj
-    if path == []
-      obj.is_a?(Array) ? obj : [obj]
-    elsif obj.is_a?(Hash)
-      key = path.shift
-      raise "Unexpected AWS Response: #{key} missing" unless
-        (obj.is_a?(Hash) && obj[key])
-
-      ensure_resp(path, obj[key])
-    else
-      obj.map{|s| ensure_resp(path.clone, s)}.flatten(1)
-    end
-  end
 end