RubyGems - marty - Versions diffs - 6.1.0 → 8.0.0 - Mend

marty 6.1.0 → 8.0.0

Files changed (89) hide show

checksums.yaml +4 -4
data/.gitignore +1 -0
data/.gitlab-ci.yml +17 -3
data/.rubocop.yml +5 -0
data/.rubocop_todo.yml +3 -2
data/Gemfile +2 -1
data/app/assets/javascripts/marty/extjs/extensions/marty.js +23 -1
data/app/assets/stylesheets/marty/application.css +4 -1
data/app/assets/stylesheets/marty/dark_mode.css +17 -0
data/app/components/marty/auth_app.rb +10 -1
data/app/components/marty/auth_app/client/auth_app.js +4 -0
data/app/components/marty/extras/layout.rb +2 -2
data/app/components/marty/extras/misc.rb +1 -1
data/app/components/marty/log_view.rb +0 -1
data/app/components/marty/main_auth_app.rb +9 -12
data/app/components/marty/promise_view.rb +5 -0
data/app/components/marty/promise_view/client/promise_view.js +11 -0
data/app/components/marty/schedule_jobs_dashboard.rb +30 -96
data/app/components/marty/schedule_jobs_grid.rb +118 -0
data/app/controllers/marty/application_controller.rb +6 -2
data/app/models/marty/base.rb +48 -48
data/app/models/marty/config.rb +14 -2
data/app/models/marty/user.rb +12 -0
data/app/services/marty/background_job/fetch_missing_in_schedule_cron_jobs.rb +19 -0
data/app/services/marty/enums/report.rb +18 -0
data/app/services/marty/promises/delorean/create.rb +16 -27
data/app/services/marty/promises/ruby/create.rb +10 -1
data/app/views/layouts/marty/application.html.erb +4 -1
data/app/views/marty/diagnostic/op.html.erb +3 -1
data/config/locales/en.yml +2 -0
data/db/migrate/512_add_promise_priority.rb +9 -0
data/db/migrate/513_add_priority_to_promise_view.rb +44 -0
data/db/migrate/514_remove_marty_events.rb +13 -0
data/delorean/enum_report.dl +11 -0
data/delorean/table_report.dl +7 -0
data/docker-compose.dummy.yml +7 -4
data/lib/marty.rb +5 -2
data/lib/marty/api/base.rb +15 -9
data/lib/marty/cache_adapters.rb +2 -0
data/lib/marty/cache_adapters/mcfly_ruby_cache.rb +1 -5
data/lib/marty/cache_adapters/memory_and_redis.rb +93 -0
data/lib/marty/cache_adapters/redis.rb +63 -0
data/lib/marty/delayed_job/scheduled_job_plugin.rb +33 -0
data/lib/marty/diagnostic/database.rb +1 -2
data/lib/marty/logger.rb +50 -17
data/lib/marty/monkey.rb +26 -6
data/lib/marty/promise_ruby_job.rb +2 -0
data/lib/marty/rails_app.rb +29 -0
data/lib/marty/railtie.rb +1 -0
data/lib/marty/version.rb +1 -1
data/marty.gemspec +1 -0
data/spec/controllers/job_controller_spec.rb +2 -2
data/spec/dummy/app/components/gemini/cm_auth_app.rb +12 -0
data/spec/dummy/app/components/gemini/simple_view.rb +17 -0
data/spec/dummy/app/jobs/test_failing_job.rb +14 -0
data/spec/dummy/app/models/gemini/helper.rb +90 -1
data/spec/dummy/config/application.rb +1 -0
data/spec/dummy/db/migrate/20191101132729_add_activity_flag_to_simple.rb +10 -0
data/spec/dummy/delorean/blame_report.dl +1 -0
data/spec/dummy/delorean/enum_report.dl +1 -0
data/spec/dummy/delorean/marty_fields.dl +1 -0
data/spec/dummy/delorean/table_report.dl +1 -0
data/spec/features/data_blame_report_spec.rb +66 -0
data/spec/features/data_grid_spec.rb +1 -1
data/spec/features/enum_values_report_spec.rb +76 -0
data/spec/features/inline_editing_spec.rb +33 -0
data/spec/features/rule_spec.rb +1 -1
data/spec/features/schedule_jobs_dashboard_spec.rb +1 -1
data/spec/features/scripting_spec.rb +1 -1
data/spec/features/user_list_report_spec.rb +74 -0
data/spec/fixtures/misc/struct_compare_tests.txt +15 -5
data/spec/job_helper.rb +39 -0
data/spec/jobs/cron_job_spec.rb +91 -0
data/spec/lib/mcfly_model_spec.rb +9 -0
data/spec/models/promise_spec.rb +168 -1
data/spec/other/diagnostic/delayed_job_workers_spec.rb +1 -1
data/spec/performance/caching_spec.rb +99 -0
data/spec/services/background_job/fetch_missing_in_schedule_cron_jobs_spec.rb +34 -0
data/spec/support/delayed_job_helpers.rb +3 -3
data/spec/support/shared_connection.rb +9 -1
data/spec/support/structure_compare.rb +19 -3
metadata +39 -6
data/app/components/marty/event_view.rb +0 -129
data/app/models/marty/event.rb +0 -317
data/spec/dummy/db/migrate/20160923183516_add_bulk_pricing_event_ops.rb +0 -8
data/spec/dummy/delorean/blame_report.dl +0 -268
data/spec/dummy/delorean/marty_fields.dl +0 -63
data/spec/dummy/delorean/table_report.dl +0 -34
data/spec/models/event_spec.rb +0 -272

data/app/views/marty/diagnostic/op.html.erb CHANGED Viewed

@@ -52,7 +52,9 @@
   <body>
     <div style="text-align:center">
       <h1 class="application <%= 'error' unless @result['errors'].empty? %>">
-        <%=ENV['DIAG_TITLE'] || Rails.application.class.parent_name%> Diagnostic
+        <%=
+          ENV['DIAG_TITLE'] || ::Marty::RailsApp.application_name
+        %> Diagnostic
       </h1>
       <h3><i><%= DateTime.now %></i></h3>
       <%== @result['display'] %>

data/config/locales/en.yml CHANGED Viewed

@@ -136,6 +136,8 @@ en:
     end_dt: End Date
     user_login: User Login
     cformat: Format
+    schedule_dashboard:
+      warnings: Warnings
   data_import_view:
     import: Import

data/db/migrate/512_add_promise_priority.rb ADDED Viewed

@@ -0,0 +1,9 @@
+class AddPromisePriority < ActiveRecord::Migration[4.2]
+  def up
+    add_column :marty_promises, :priority, :integer
+  end
+  def down
+    remove_column :marty_promises, :priority
+  end
+end

data/db/migrate/513_add_priority_to_promise_view.rb ADDED Viewed

@@ -0,0 +1,44 @@
+class AddPriorityToPromiseView < ActiveRecord::Migration[4.2]
+  def up
+    execute <<~SQL
+       DROP VIEW IF EXISTS marty_vw_promises;
+       CREATE OR REPLACE VIEW marty_vw_promises
+       AS
+       SELECT
+       id,
+       title,
+       user_id,
+       cformat,
+       parent_id,
+       job_id,
+       status,
+       start_dt,
+       end_dt,
+       priority
+       FROM marty_promises;
+       GRANT SELECT ON marty_vw_promises TO public;
+    SQL
+  end
+  def down
+    execute <<~SQL
+       DROP VIEW IF EXISTS marty_vw_promises;
+       CREATE OR REPLACE VIEW marty_vw_promises
+       AS
+       SELECT
+       id,
+       title,
+       user_id,
+       cformat,
+       parent_id,
+       job_id,
+       status,
+       start_dt,
+       end_dt
+       FROM marty_promises;
+       GRANT SELECT ON marty_vw_promises TO public;
+    SQL
+  end
+end

data/db/migrate/514_remove_marty_events.rb ADDED Viewed

@@ -0,0 +1,13 @@
+class RemoveMartyEvents < ActiveRecord::Migration[4.2]
+  def up
+    drop_table :marty_events
+    execute <<-SQL
+      DROP TYPE enum_event_operations;
+    SQL
+  end
+  def down
+    announce("No-op on RemoveMartyEvents.down")
+  end
+end

data/delorean/enum_report.dl ADDED Viewed

@@ -0,0 +1,11 @@
+import MartyFields
+import Styles
+EnumValuesReport:
+    title   = "Enum Values List"
+    roles   = ['admin', 'dev', 'data_grid_editor']
+    raw     = Marty::Enums::Report.call()
+    result  = Marty::Helper.to_csv(raw)
+    form    = []
+    format  = "csv"

data/delorean/table_report.dl CHANGED Viewed

@@ -32,3 +32,10 @@ TableReport:
         ]
     format      = "csv"
+UserReport:
+    title   = "User List"
+    roles   = ['admin', 'user_manager']
+    raw     = Marty::User.export_for_report()
+    result  = Marty::Helper.to_csv(raw)
+    form    = []
+    format  = "csv"

data/docker-compose.dummy.yml CHANGED Viewed

@@ -17,9 +17,11 @@ services:
       - "HEADLESS=true"
       - "PGTZ=America/Los_Angeles"
       - "BUNDLER_VERSION=2.0.1"
+      - "MARTY_REDIS_URL=redis:6379/1"
     # env_file: ".rbenv-vars"
     depends_on:
       - "postgres"
+      - "redis"
     volumes:
       - .:/opt/app:delegated
       - '.bash_history.docker:/root/.bash_history'
@@ -34,12 +36,13 @@ services:
     volumes:
       - postgresql-data:/var/lib/postgresql/data
-  bundle_box:
-    image: busybox
+  redis:
+    image: 'redis:5.0.6-alpine'
     volumes:
-      - /bundle_box
+      - redis-data:/data
 volumes:
-  postgresql-data:
   bundle_box:
+  postgresql-data:
+  redis-data:

data/lib/marty.rb CHANGED Viewed

@@ -15,6 +15,7 @@ require 'marty/cache_adapters'
 require 'marty/monkey'
 require 'marty/promise_job'
 require 'marty/json_schema'
+require 'marty/rails_app'
 # This does not get loaded in via bundler unless it is included in the
 # application's Gemfile. Requiring it here removes the need to add it
@@ -25,7 +26,9 @@ require 'delayed_cron_job'
 require 'pathname'
 module Marty
-  def self.root
-    Pathname.new(File.expand_path('..', __dir__))
+  class << self
+    def root
+      Pathname.new(File.expand_path('..', __dir__))
+    end
   end
 end

data/lib/marty/api/base.rb CHANGED Viewed

@@ -53,14 +53,19 @@ class Marty::Api::Base
     params = params.deep_dup
     schema_key = [params[:tag], params[:script], params[:node], params[:attr]]
-    input_schema = nil
-    begin
-      # get_schema will either return a hash with the schema,
-      # or a string with the error
-      input_schema = @@schemas[schema_key] ||=
-        Marty::JsonSchema.get_schema(*schema_key)
-    rescue StandardError => e
-      return { error: e.message }
+    input_schema = @@schemas[schema_key]
+    unless input_schema
+      begin
+        # get_schema will either return a hash with the schema,
+        # or a string with the error
+        result_schema = Marty::JsonSchema.get_schema(*schema_key)
+        # only store schema in cache when not error
+        @@schemas[schema_key] = result_schema if result_schema.is_a?(Hash)
+        input_schema = result_schema
+      rescue StandardError => e
+        return { error: e.message }
+      end
     end
     # if schema was found
@@ -165,7 +170,8 @@ class Marty::Api::Base
   def self.filter_hash hash, filter_params
     return unless hash
-    pf = ActionDispatch::Http::ParameterFilter.new(filter_params)
+    pf_class = ::Marty::RailsApp.parameter_filter_class
+    pf = pf_class.new(filter_params)
     pf.filter(hash.stringify_keys)
   end

data/lib/marty/cache_adapters.rb CHANGED Viewed

@@ -1,4 +1,6 @@
 require 'marty/cache_adapters/mcfly_ruby_cache'
+require 'marty/cache_adapters/memory_and_redis'
+require 'marty/cache_adapters/redis'
 module Marty
   module CacheAdapters

data/lib/marty/cache_adapters/mcfly_ruby_cache.rb CHANGED Viewed

@@ -2,11 +2,7 @@ module Marty
   module CacheAdapters
     class McflyRubyCache < ::Delorean::Cache::Adapters::RubyCache
       def cache_item?(klass:, method_name:, args:)
-        ts = args && args.first
-        return false if Mcfly.is_infinity(ts)
-        true
+        !Mcfly.is_infinity(args&.first)
       end
     end
   end

data/lib/marty/cache_adapters/memory_and_redis.rb ADDED Viewed

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+module Marty
+  module CacheAdapters
+    class MemoryAndRedis < ::Delorean::Cache::Adapters::Base
+      POST = '__RedisCache'
+      def initialize(
+        size_per_class: 1000,
+        redis_url: Rails.application.config.marty.redis_url,
+        redis_expires_in: 48.hours
+      )
+        @size_per_class = size_per_class
+        @redis_adapter = ::Marty::CacheAdapters::Redis.new(
+          redis_url: redis_url,
+          expires_in: redis_expires_in
+        )
+        @memory_adapter = ::Marty::CacheAdapters::McflyRubyCache.new(
+          size_per_class: size_per_class
+        )
+      end
+      def cache_item(klass:, cache_key:, item:)
+        @redis_adapter.cache_item(
+          klass: klass,
+          cache_key: cache_key,
+          item: item
+        )
+        @memory_adapter.cache_item(
+          klass: klass,
+          cache_key: cache_key,
+          item: item
+        )
+      end
+      # When cache is found in local memory, we simply return the cached item.
+      # Otherwise we look into Redis, if item is cached there,
+      # we copy it to local memory to speed up future lookups.
+      def fetch_item(klass:, cache_key:, default: nil)
+        memory_item = @memory_adapter.fetch_item(
+          klass: klass,
+          cache_key: cache_key,
+          default: default
+        )
+        return memory_item if memory_item != default
+        redis_item = @redis_adapter.fetch_item(
+          klass: klass,
+          cache_key: cache_key,
+          default: default
+        )
+        return default if redis_item == default
+        @memory_adapter.cache_item(
+          klass: klass,
+          cache_key: cache_key,
+          item: redis_item
+        )
+        redis_item
+      end
+      def cache_key(klass:, method_name:, args:)
+        r = ["#{klass.name}#{POST}", method_name] + args.map do |arg|
+          arg.respond_to?(:id) ? arg.id : arg
+          arg
+        end.freeze
+        Marshal.dump r
+      end
+      def clear!(klass:)
+        @redis_adapter.clear!(klass: klass)
+        @memory_adapter.clear!(klass: klass)
+      end
+      def clear_all!
+        @redis_adapter.clear_all!
+        @memory_adapter.clear_all!
+      end
+      def cache_item?(klass:, method_name:, args:)
+        !Mcfly.is_infinity(args&.first)
+      end
+    end
+  end
+end

data/lib/marty/cache_adapters/redis.rb ADDED Viewed

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+require 'redis'
+module Marty
+  module CacheAdapters
+    class Redis < ::Delorean::Cache::Adapters::Base
+      POST = '__RedisCache'
+      def initialize(
+        size_per_class: 1000,
+        redis_url: Rails.application.config.marty.redis_url,
+        expires_in: 48.hours
+      )
+        @redis = ::Redis.new(url: "redis://#{redis_url}")
+        @expires_in = expires_in
+      end
+      def cache_item(klass:, cache_key:, item:)
+        @redis.set(
+          cache_key,
+          Marshal.dump(item),
+          ex: @expires_in.seconds.to_i
+        )
+      end
+      def fetch_item(klass:, cache_key:, default: nil)
+        r = @redis.get(cache_key)
+        return default if r.nil?
+        Marshal.load(r)
+      end
+      def cache_key(klass:, method_name:, args:)
+        r = ["#{klass.name}#{POST}", method_name] + args.map do |arg|
+          next arg.id if arg.respond_to?(:id)
+          arg
+        end.freeze
+        Marshal.dump r
+      end
+      def clear!(klass:)
+        keys = @redis.keys("*#{klass.name}#{POST}*")
+        @redis.pipelined do
+          keys.each do |key|
+            @redis.del key
+          end
+        end
+      end
+      def clear_all!
+        @redis.flushall
+      end
+      def cache_item?(klass:, method_name:, args:)
+        !Mcfly.is_infinity(args&.first)
+      end
+    end
+  end
+end

data/lib/marty/delayed_job/scheduled_job_plugin.rb ADDED Viewed

@@ -0,0 +1,33 @@
+module Marty
+  module DelayedJob
+    class ScheduledJobPlugin < Delayed::Plugin
+      class << self
+        def cron?(job)
+          job.cron.present?
+        end
+      end
+      callbacks do |lifecycle|
+        # We want to nullify cron column if job schedule was turned off
+        # while the job was running, so it won't add a new record to
+        # delayed_jobs table
+        lifecycle.before(:error) do |worker, job, &block|
+          if cron?(job)
+            begin
+              job_class_str = job.handler.split("\n").find do |line|
+                line.include? 'job_class'
+              end
+              job_class_name = job_class_str.gsub('job_class:', '').strip
+              job_class = job_class_name.constantize
+              job.cron = job_class.cron_expression
+            rescue StandardError
+            end
+          else
+            # No cron job - proceed as normal
+            block.call(worker, job)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/marty/diagnostic/database.rb CHANGED Viewed

@@ -22,8 +22,7 @@ module Marty::Diagnostic::Database
   def self.db_schema
     current = ActiveRecord::Migrator.current_version
     raise "Migration is needed.\nCurrent Version: #{current}" if
-      Rails.version >= '5.2.0' &&
-      ActiveRecord::Base.connection.migration_context.needs_migration?
+        ::Marty::RailsApp.needs_migration?
     current.to_s
   end

data/lib/marty/logger.rb CHANGED Viewed

@@ -1,27 +1,60 @@
 class Marty::Logger
   include Delorean::Model
-  def self.method_missing(m, *args, &block)
-    return super unless
-      [:debug, :info, :warn, :error, :fatal, :unknown].include?(m)
-    Marty::Util.logger.send(m, args[0]) if Marty::Util.logger.respond_to?(m)
-    log(m, *args)
+  delorean_fn :dllog, sig: [2, 20] do |*args|
+    info args[0], args[1..-1]
   end
-  def self.log(type, message, details = nil)
-    Marty::Log.write_log(type, message, details)
-  end
+  class << self
+    def log_event(event_name, *args)
+      if Marty::Util.logger.respond_to?(event_name)
+        Marty::Util.logger.send(
+          event_name,
+          args[0]
+        )
+      end
+      log(event_name, *args)
+    end
+    def debug(*args)
+      log_event(:debug, *args)
+    end
+    def info(*args)
+      log_event(:info, *args)
+    end
+    def warn(*args)
+      log_event(:warn, *args)
+    end
+    def error(*args)
+      log_event(:error, *args)
+    end
-  def self.with_logging(error_message, error_data)
+    def fatal(*args)
+      log_event(:fatal, *args)
+    end
+    def unknown(*args)
+      log_event(:unknown, *args)
+    end
+    def log(type, message, details = nil)
+      Marty::Log.write_log(type, message, details)
+    end
+    def with_logging(error_message, error_data)
       yield
-  rescue StandardError => e
-      error(error_message,  'message' => e.message,
-                             'data' => error_data)
-      raise "#{error_message}: #{e.message}"
-  end
+    rescue StandardError => e
+      error(
+        error_message,
+        'message' => e.message,
+        'data' => error_data
+      )
-  delorean_fn :dllog, sig: [2, 20] do |*args|
-     info args[0], args[1..-1]
+      raise "#{error_message}: #{e.message}"
+    end
   end
 end