marty 5.2.0 → 6.1.0

data/Dockerfile.dummy

@@ -1,6 +1,6 @@
 # ONLY FOR DEVELOPMENT PURPOSES
 # dockerfile is not suitable to produce a production-grade docker image
-FROM ruby:2.5.1
+FROM ruby:2.6.3
 
 WORKDIR /opt/app
 

data/app/assets/javascripts/marty/extjs/extensions/marty.js

@@ -897,3 +897,14 @@ Ext.define('Marty.layout.container.Auto', {
     },
 
 });
+
+Ext.define('overrides.grid.column.Column', {
+    override: 'Ext.grid.column.Column',
+
+    initConfig: function(config) {
+        if (!config.renderer && !this.updater) {
+            config.formatter = 'htmlEncode'
+        }
+        return this.callParent(arguments)
+    }
+  })

data/app/components/marty/log_view.rb

@@ -59,7 +59,7 @@ class Marty::LogView < Marty::Grid
     c.read_only    = true
     c.filterable = true
     c.xtype        = :datecolumn
-    c.format       = 'Y-m-d h:i:s.u'
+    c.format       = 'Y-m-d H:i:s.u'
     c.field_config = {
       xtype: :displayfield,
     }

data/app/components/marty/promise_view.rb

@@ -46,9 +46,6 @@ class Marty::PromiseView < Netzke::Tree::Base
       update: class_can?(:update),
       delete: class_can?(:delete)
     }
-
-    # garbage collect old promises (hacky to do this here)
-    Marty::Promise.cleanup(false)
   end
 
   def bbar

data/app/jobs/marty/remove_old_promises_job.rb

@@ -0,0 +1,7 @@
+module Marty
+  class RemoveOldPromisesJob < ::Marty::CronJob
+    def perform
+      Marty::Promise.cleanup(false)
+    end
+  end
+end

data/app/models/marty/data_grid.rb

@@ -226,12 +226,6 @@ class Marty::DataGrid < Marty::Base
             "ri: #{row_info}" if res['error']
     end
 
-    if ret_grid_data
-      dg = find(dgh['id'])
-      md, mmd = modify_grid(h_passed, dg.metadata, dg.data)
-      res['data'] = md
-      res['metadata'] = mmd
-    end
     res
   end
 
@@ -683,142 +677,8 @@ class Marty::DataGrid < Marty::Base
     end
   end
 
-  def self.modify_grid(params, metadata, data)
-    removes = ['h', 'v'].each_with_object({}) { |dir, hash| hash[dir] = Set.new }
-
-    metadata_copy, data_copy = metadata.deep_dup, data.deep_dup
-
-    metadata_copy.each do |meta|
-      dir, keys, type, rs_keep = meta.values_at(
-        'dir', 'keys', 'type', 'rs_keep')
-      next unless rs_keep
-
-      if type == 'numrange' || type == 'int4range'
-        modop, modvalparm = parse_bounds(rs_keep)
-        modval = params[modvalparm]
-        if modval
-          prune_a, rewrite_a = compute_numeric_mods(keys, modop, modval)
-          removes[dir].merge(prune_a)
-          rewrite_a.each { |(ind, value)| keys[ind] = value }
-        end
-      else
-        modval = params[rs_keep]
-        if modval
-          prune_a, rewrite_a = compute_set_mods(keys, modval)
-          removes[dir].merge(prune_a)
-          rewrite_a.each { |(ind, value)| keys[ind] = value }
-        end
-      end
-    end
-
-    removes.reject! { |_dir, set| set.empty? }
-
-    removes.each do |dir, _set|
-      metadata_copy.select { |m| m['dir'] == dir }.each do |meta|
-        meta['keys'] = remove_indices(meta['keys'], removes[dir])
-      end
-    end
-
-    data_copy = remove_indices(data_copy, removes['v']) if removes['v']
-
-    data_copy.each_index do |index|
-      data_copy[index] = remove_indices(data_copy[index], removes['h'])
-    end if removes['h']
-
-    [data_copy, metadata_copy]
-  end
-
   private
 
-  def self.remove_indices(orig_array, inds)
-    orig_array.each_with_object([]).with_index do |(item, new_array), index|
-      new_array.push(item) unless inds.include?(index)
-    end
-  end
-
-  def self.opposite_sign(op)  # toggle sign and inclusivity
-    {
-      :<  => :>=,
-      :<= => :>,
-      :>  => :<=,
-      :>= => :<,
-    }[op]
-  end
-
-  def self.compute_numeric_mods(keys, op, val)
-    @keyhash ||= {}
-    prune_a, rewrite_a = [], []
-
-    # features allow multiple values, but for constraint on a grid range
-    # only a scalar is meaningful.  so if there are multiple values we
-    # take the first value to use
-    value = val.is_a?(Array) ? val[0] : val
-    keys.each_with_index do |key, index|
-      lhop, orig_lhv, orig_rhv, rhop = @keyhash[key] ||= parse_range(key)
-
-      lhv, rhv = orig_lhv || -Float::INFINITY, orig_rhv || Float::INFINITY
-
-      case op
-      when :>=, :>
-        next if value > rhv
-
-        if value == rhv
-          if rhop == :<= && op == :>=
-            rewrite_a.push(
-              [index, rewrite_range(lhop, orig_lhv, orig_rhv, :<)])
-          end
-        elsif value > lhv
-          rewrite_a.push(
-            [index, rewrite_range(lhop, orig_lhv, value, opposite_sign(op))])
-        elsif value == lhv && lhop == :>= && op == :>
-          rewrite_a.push([index, rewrite_range(:>=, value, value, :<=)])
-        elsif value <= lhv
-          prune_a.push(index)
-        end
-      when :<=, :<
-        next if value < lhv
-
-        if value == lhv
-          if lhop == :>= && op == :<=
-            rewrite_a.push(
-              [index, rewrite_range(:>, orig_lhv, orig_rhv, rhop)])
-          end
-        elsif value < rhv
-          rewrite_a.push(
-            [index, rewrite_range(opposite_sign(op), value, orig_rhv, rhop)])
-        elsif value == rhv && rhop == :<= && op == :<
-          rewrite_a.push([index, rewrite_range(:>=, value, value, :<=)])
-        elsif value >= rhv
-          prune_a.push(index)
-        end
-      end
-    end
-    [prune_a, rewrite_a]
-  end
-
-  # value is a list of what to keep
-  def self.compute_set_mods(keys, val)
-    prune_a, rewrite_a, value = [], [], Array(val)
-
-    keys.each_with_index do |key, index|
-      # rewrite any nil (wildcard) keys in the dimension
-      # to be our 'to-keep' val(s)
-      if key.nil?
-        rewrite_a.push([index, value])
-        next
-      end
-
-      remove = key - value
-      if remove == key
-        prune_a.push(index)
-        next
-      end
-
-      rewrite_a.push([index, key - remove]) if remove != []
-    end
-    [prune_a, rewrite_a]
-  end
-
   def self.parse_range(key)
     match = key.match(/\A(\[|\()([0-9\.-]*),([0-9\.-]*)(\]|\))\z/)
     raise "unrecognized pattern #{key}" unless match
@@ -831,29 +691,6 @@ class Marty::DataGrid < Marty::Base
     [lboundary == '(' ? :> : :>=, lhv, rhv, rboundary == ')' ? :< : :<=]
   end
 
-  def self.rewrite_range(lb, lhv, rhv, rb)
-    lboundary = lb == :> ? '(' : '['
-
-    # even though numranges are float type, we don't want to output ".0"
-    # for integer values.  So for values like that we convert to int
-    # first before conversion to string
-    lvalue = (lhv.to_i == lhv ? lhv.to_i : lhv).to_s
-    rvalue = (rhv.to_i == rhv ? rhv.to_i : rhv).to_s
-    rboundary = rb == :< ? ')' : ']'
-    lboundary + lvalue + ',' + rvalue + rboundary
-  end
-
-  def self.parse_bounds(key)
-    match = key.match(/\A *(<|>|<=|>=)? *([a-z_]+) *\z/)
-    raise "unrecognized pattern #{key}" unless match
-
-    opstr, ident = match[1..2]
-
-    # data grid value is expressed as what to keep
-    # we convert to the opposite (what to prune)
-    [opposite_sign(opstr.to_sym), ident]
-  end
-
   def self.remove_not(string)
     return string unless string.starts_with?(NOT_STRING_START)
     return string unless string.ends_with?(NOT_STRING_END)

data/app/models/marty/event.rb

@@ -119,10 +119,10 @@ SQL
   end
 
   def self.last_event(klass, subject_id, operation = nil)
-    hash = all_running.select do |pm|
+    hash = all_running.reverse.find do |pm|
       pm['klass'] == klass && pm['subject_id'] == subject_id.to_i &&
         (operation.nil? || pm['enum_event_operation'] == operation)
-    end.last
+    end
 
     return hash if hash
 

data/app/models/marty/posting.rb

@@ -67,13 +67,6 @@ class Marty::Posting < Marty::Base
     q.order('created_dt DESC').first.attributes
   end
 
-  def self.get_latest(limit, _is_test = nil)
-    # IMPORTANT: is_test arg is ignored (KEEP for backward compat.)
-
-    q = where("created_dt <> 'infinity'").
-       order('created_dt DESC').limit(limit)
-  end
-
   delorean_fn :get_latest_by_type, sig: [1, 2] do |limit, posting_types = []|
     raise 'missing posting types list' unless posting_types
     raise 'bad posting types list' unless posting_types.is_a?(Array)

data/app/models/marty/promise.rb

@@ -213,7 +213,7 @@ class Marty::Promise < Marty::Base
     def cleanup(all = false)
       where(
         'start_dt < ? AND parent_id IS NULL',
-        DateTime.now - (all ? 0.hours : 4.hours)
+        all ? Time.zone.now : 4.hours.ago
       ).destroy_all
     rescue StandardError => e
         Marty::Util.logger.error("promise GC error: #{e}")

data/app/services/marty/data_grid_view/save_grid.rb

@@ -25,8 +25,8 @@ module Marty
         data_as_array = data.map do |row|
           row.keys.map { |key| row[key] }
         end
-        vcnt = dg.metadata.select { |md| md['dir'] == 'v' }.count
-        hcnt = dg.metadata.select { |md| md['dir'] == 'h' }.count
+        vcnt = dg.metadata.count { |md| md['dir'] == 'v' }
+        hcnt = dg.metadata.count { |md| md['dir'] == 'h' }
         cur_data_dim = [dg.data.length, dg.data[0].length]
         exported = dg.export.lines
         sep = exported.each_with_index.detect { |l, _i| /^\s*$/.match(l) }.last

data/app/services/marty/jobs/schedule.rb

@@ -7,6 +7,11 @@ module Marty
         glob = Rails.root.join('app', 'jobs', '**', '*_job.rb')
         Dir.glob(glob).each { |f| require f }
 
+        glob2 = Marty.root.join('app', 'jobs', '**', '*_job.rb')
+        Dir.glob(glob2).each { |f| require f }
+
+        Delayed::Job.where.not(cron: nil).each(&:destroy!)
+
         Marty::CronJob.subclasses.map do |klass|
           klass.schedule
           [klass.name, klass.cron_expression]

data/db/migrate/510_schedule_job_to_remove_old_promises.rb

@@ -0,0 +1,19 @@
+class ScheduleJobToRemoveOldPromises < ActiveRecord::Migration[4.2]
+  def up
+    cron_every_hour = '0 * * * *'
+
+    Marty::BackgroundJob::Schedule.create!(
+      job_class: 'Marty::RemoveOldPromisesJob',
+      cron: cron_every_hour,
+      state: 'on'
+    )
+
+    ::Marty::RemoveOldPromisesJob.schedule
+  end
+
+  def down
+    Marty::BackgroundJob::Schedule.find_by(
+      job_class: 'Marty::RemoveOldPromisesJob'
+    )&.destroy
+  end
+end

data/lib/marty.rb

@@ -21,3 +21,11 @@ require 'marty/json_schema'
 # to the Gemfile
 require 'net-ldap'
 require 'delayed_cron_job'
+
+require 'pathname'
+
+module Marty
+  def self.root
+    Pathname.new(File.expand_path('..', __dir__))
+  end
+end

data/{other → lib}/marty/api/base.rb

@@ -28,11 +28,9 @@ class Marty::Api::Base
     params
   end
 
-  def self.before_evaluate api_params
-  end
+  def self.before_evaluate api_params; end
 
-  def self.after_evaluate api_params, result
-  end
+  def self.after_evaluate api_params, result; end
 
   @@numbers = {}
   @@schemas = {}
@@ -60,7 +58,7 @@ class Marty::Api::Base
       # get_schema will either return a hash with the schema,
       # or a string with the error
       input_schema = @@schemas[schema_key] ||=
-                       Marty::JsonSchema.get_schema(*schema_key)
+        Marty::JsonSchema.get_schema(*schema_key)
     rescue StandardError => e
       return { error: e.message }
     end
@@ -69,7 +67,7 @@ class Marty::Api::Base
     if input_schema.is_a?(Hash)
       # fix numbers types
       numbers = @@numbers[schema_key] ||=
-                  Marty::JsonSchema.get_numbers(input_schema)
+        Marty::JsonSchema.get_numbers(input_schema)
 
       # modify params in place
       Marty::JsonSchema.fix_numbers(params[:params], numbers)
@@ -85,14 +83,14 @@ class Marty::Api::Base
       return { "error": input_schema } if input_schema.is_a?(String)
 
       begin
-        res = SchemaValidator::validate_schema(input_schema, params[:params])
+        res = SchemaValidator.validate_schema(input_schema, params[:params])
       rescue NameError
         return { error: "Unrecognized PgEnum for attribute #{params[:attr]}" }
       rescue StandardError => e
         return { error: "#{params[:attr]}: #{e.message}" }
       end
 
-      schema_errors = SchemaValidator::get_errors(res) unless res.empty?
+      schema_errors = SchemaValidator.get_errors(res) unless res.empty?
       return { error: "Error(s) validating: #{schema_errors}" } if
         schema_errors
     end
@@ -101,8 +99,8 @@ class Marty::Api::Base
     begin
       engine = Marty::ScriptSet.new(params[:tag]).get_engine(params[:script])
     rescue StandardError => e
-      error = "Can't get engine: #{params[:script] || 'nil'} with tag: " +
-                "#{params[:tag] || 'nil'}; message: #{e.message}"
+      error = "Can't get engine: #{params[:script] || 'nil'} with tag: " \
+              "#{params[:tag] || 'nil'}; message: #{e.message}"
       Marty::Logger.info error
       return { error: error }
     end
@@ -127,13 +125,13 @@ class Marty::Api::Base
       if config[:output_validated] && !(res.is_a?(Hash) && res['error'])
         begin
           output_schema_params = params + { attr: params[:attr] + '_' }
-          schema = SchemaValidator::get_schema(output_schema_params)
+          schema = SchemaValidator.get_schema(output_schema_params)
         rescue StandardError => e
           return { error: e.message }
         end
 
         begin
-          schema_errors = SchemaValidator::validate_schema(schema, res)
+          schema_errors = SchemaValidator.validate_schema(schema, res)
         rescue NameError
           return { error: "Unrecognized PgEnum for attribute #{attr}" }
         rescue StandardError => e
@@ -160,7 +158,7 @@ class Marty::Api::Base
       Marty::Logger.info "Evaluation error: #{msg}"
       return retval = msg
     ensure
-      error = Hash === retval ? retval[:error] : nil
+      error = retval.is_a?(Hash) ? retval[:error] : nil
     end
   end
 
@@ -189,8 +187,7 @@ class Marty::Api::Base
       end_time:   Time.zone.now,
       error:      error,
       remote_ip:  request.remote_ip,
-      auth_name:  params[:auth]
-    }
+      auth_name:  params[:auth] }
   end
 
   def self.log result, params, request
@@ -221,7 +218,7 @@ class Marty::Api::Base
     end
 
     def self.massage_message(msg)
-      m = %r|'#/([^']+)' of type ([^ ]+) matched the disallowed schema|.
+      m = %r{'#/([^']+)' of type ([^ ]+) matched the disallowed schema}.
             match(msg)
 
       return msg unless m
@@ -239,7 +236,7 @@ class Marty::Api::Base
           fa, fragment, message, errors = errs.values_at(:failed_attribute,
                                                          :fragment,
                                                          :message, :errors)
-          ((['AllOf', 'AnyOf', 'Not'].include?(fa) && fragment == '#/') ?
+          (['AllOf', 'AnyOf', 'Not'].include?(fa) && fragment == '#/' ?
              [] : [massage_message(message)]) + _get_errors(errors || {})
         end
       end

data/{other → lib}/marty/diagnostic/aws/ec2_instance.rb

@@ -70,13 +70,13 @@ class Marty::Diagnostic::Aws::Ec2Instance < Marty::Aws::Request
     instances = ensure_resp(
       ['reservationSet', 'item', 'instancesSet', 'item'],
       resp
-    ).map do |i|
+    ).flat_map do |i|
       {
         'id'    => i['instanceId'],
         'ip'    => i['privateIpAddress'],
         'state' => i['instanceState'],
       }
-    end.flatten(1)
+    end
   end
 
   def get_private_ips