marty 14.0.0 → 14.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 268b89d12df4e9698333655172962a7ae5cdb083298bbf3260d5e9314527a4cf
-  data.tar.gz: 033e4df522a0627af012eb84174cbad90e556e055a0a0d29383dd33dc13aa65f
+  metadata.gz: fd079baae110bb7b7b5fdfefa0cb1c5a1c03779cbab01c8ff3f92e587cb1e6e3
+  data.tar.gz: fdf66588c81dee9eec99458c29ea3a519cb454b40340ba344a3daab42da2544d
 SHA512:
-  metadata.gz: b936825cf18110a8a02691a02f35a1c8c74eb655d1ceed88600d16e2aeb8543cdb7d4e5d36fe026a0d9947d15a556cee7b4c0e6518878ddf90f7de792633d6a8
-  data.tar.gz: 9d447d0881e640da79cdbcdbce23d9ce01ab851722c8decde17d944f48e3fd5799df4ed35ee04aa983c083741cf9b391b512b343fd3d5cee8dc7839b3b3b2488
+  metadata.gz: ad18ce255689328fbdb69490b4b47fbcdd97866953e92d7a68e838e2185443116539fbd65c16cc84c30ed9d9d08f2c8399b054cdced48769f48d414f523f5463
+  data.tar.gz: 1dd8b6f97729917f3e6dece1c20464eb01f3617fd6c5862cde4d897714680aee950bc255c4287f6b2c90183690d156978d11eefa2e1a2b86fda71a177efbf3b8

data/.eslintignore

@@ -1,3 +1,4 @@
 app/assets/javascripts/marty/codemirror/*
 !.schemalintrc.js
+!.schemalint-rules/*
 !.eslintrc.js

data/.eslintrc.js

@@ -10,7 +10,8 @@ module.exports = {
     Ext: "readonly",
     CodeMirror: "readonly",
     module: "writable",
-    process: "readonly"
+    process: "readonly",
+    require: "readonly"
   },
   parserOptions: {
     ecmaVersion: 6

data/.schemalint-rules/index.js

@@ -0,0 +1,5 @@
+const preferVarcharWithoutSizeLimit = require("./preferVarcharWithoutSizeLimit");
+
+module.exports = {
+  preferVarcharWithoutSizeLimit
+};

data/.schemalint-rules/preferVarcharWithoutSizeLimit.js

@@ -0,0 +1,29 @@
+const preferVarcharWithoutSizeLimit = {
+  name: "prefer-varchar-without-size-limit",
+  docs: {
+    description: "Prefer using VARCHAR without size limit over varchar(255)",
+    url: "..."
+  },
+
+  process({ schemaObject, report }) {
+    const validator = ({ name: tableName }) => (column) => {
+      const columnName = column.name;
+      const type = column.type;
+
+      if (type.startsWith("varchar") && column.maxLength) {
+        report({
+          rule: this.name,
+          identifier: `${schemaObject.name}.${tableName}.${columnName}`,
+          message: `Prefer varchar to ${type}(${column.maxLength}) types`,
+          suggestedMigration: `ALTER TABLE "${tableName}" ALTER COLUMN "${columnName}" TYPE VARCHAR;`
+        });
+      }
+    };
+
+    schemaObject.tables.forEach((table) => {
+      table.columns.forEach(validator(table));
+    });
+  }
+};
+
+module.exports = preferVarcharWithoutSizeLimit;

data/.schemalintrc.js

@@ -7,16 +7,13 @@ module.exports = {
     charset: "utf8"
   },
 
-  // plugins: ['./custom-rules'],
+  plugins: ["./.schemalint-rules"],
 
   rules: {
     "name-casing": ["error", "snake"],
     "name-inflection": ["error", "plural"],
-    "prefer-jsonb-to-json": ["error"]
-    // FIXME: user varchar with no size limit instead
-    // We would need to update lib so it would return column size info
-    // And create our own rule that checks that
-    // "prefer-text-to-varchar": ["error"]
+    "prefer-jsonb-to-json": ["error"],
+    "prefer-varchar-without-size-limit": ["error"]
   },
 
   schemas: [{ name: "public" }],

data/CHANGELOG.md

@@ -1,3 +1,27 @@
+14.2.0 - 2020-05-05
+=====================================================
+* Treating passed nil to data grids in the same way as missing attribute
+broke our lookups. Roll that change back. With this change, the behavior
+would be the following:
+
+In non strict null mode:
+missing attribute matches everything
+passed nil matches only wildcard keys (empty keys)
+
+In strict_null_mode:
+missing attribute matches only NULLs and wildcard keys
+passed nil matches only NULLs and wildcard keys
+
+14.2.0 - 2020-05-05
+=====================================================
+* Adds `Marty::Diagnostic::Version.git_tag` method that is used in diags and can be redifined in Marty apps.
+
+14.1.0 - 2020-05-01
+=====================
+* Add cleaner service and migration. The job is disabled by default.
+
+In monkey.rb override `CLASSES_TO_CLEAN` with the desired classes to be scanned/cleaned.
+
 14.0.0 - 2020-04-28
 =====================================================
 * Adds NULL support for data grid matchers:
@@ -9,6 +33,10 @@ NULL can be combined with other values in array
 
 * DataGrid's PLPGSQL lookups are no longer supported
 
+13.2.0 - 2020-04-30
+=====================
+* Add PDF content type handling
+
 13.1.0 - 2020-04-14
 =====================
 * Use ruby for Marty::DataGrid lookups. That gives a small performance boost and simplifies the code.

data/app/jobs/marty/cleaner_job.rb

@@ -0,0 +1,16 @@
+module Marty
+  class CleanerJob < ::Marty::CronJob
+    def perform
+      system_login = Rails.configuration.marty.system_account
+      Marty::Logger.info("Starting CleanerJob as user: #{system_login}")
+      system_user = Marty::User.find_by(login: system_login)
+
+      Marty::Promises::Ruby::Create.call(
+        module_name: 'Marty::Cleaner::CleanAll',
+        method_name: 'call',
+        method_args: [],
+        params: { _user_id: system_user&.id }
+      )
+    end
+  end
+end

data/app/models/marty/data_grid.rb

@@ -201,13 +201,15 @@ class Marty::DataGrid < Marty::Base
 
       unless dgh['strict_null_mode']
         next unless h_passed.key?(attr)
-        # FIXME: Make sure it won't break lookups
+
+        # FIXME: Treating passed nil in the same way
+        # as missing broke our lookups. Maybe we should get back to it later.
         # Before missing attribute would match anything,
         # while explicitly passed nil would only match wildcard keys
         # We want to be consistent and treat nil attribute as missing one,
         # unless it's a stict_null_mode, where nil would be explicitly mapped
         # to NULL keys
-        next if val.nil?
+        # next if val.nil?
       end
 
       converted_val = if val.nil?
@@ -240,7 +242,7 @@ class Marty::DataGrid < Marty::Base
                       checks.all? do |check|
                         converted_val.send(check[0], check[1])
                       end
-                    elsif key_val.nil? # Non-wildcard lookup
+                    elsif key_val.nil?
                       val.nil?
                     elsif m_type == 'boolean'
                       key_val == converted_val

data/app/services/marty/cleaner/clean_all.rb

@@ -0,0 +1,39 @@
+module Marty
+  module Cleaner
+    module CleanAll
+      LOG_DAYS_KEY = 'log_days'
+      MCFLY_DAYS_KEY = 'mcfly_days'
+      TS_DAYS_KEY = 'timestamp_days'
+
+      class << self
+        def log(table_type, table_name)
+          ::Marty::Logger.log(table_type, 'Start Clean', table_name)
+          count = yield
+          ::Marty::Logger.log(
+            table_type,
+            'End Clean',
+            "#{table_name} (#{count} records deleted)"
+          )
+        end
+
+        def call
+          window_config = ::Marty::Cleaner::MaintenanceWindow.call
+          log_days = window_config.fetch(LOG_DAYS_KEY, 60)
+          mcfly_days = window_config.fetch(MCFLY_DAYS_KEY, 365 * 3)
+          ts_days = window_config.fetch(TS_DAYS_KEY, 365 * 3)
+
+          [
+            [LOG_DAYS_KEY, log_days],
+            [MCFLY_DAYS_KEY, mcfly_days],
+            [TS_DAYS_KEY, ts_days]
+          ].each do |key, value|
+            raise "'#{key}' must be an integer" unless value.is_a?(Integer)
+          end
+          ::Marty::Cleaner::Logs.call(log_days)
+          ::Marty::Cleaner::McflyModels.call(mcfly_days)
+          ::Marty::Cleaner::TimestampModels.call(ts_days)
+        end
+      end
+    end
+  end
+end

data/app/services/marty/cleaner/logs.rb

@@ -0,0 +1,13 @@
+module Marty
+  module Cleaner
+    module Logs
+      class << self
+        def call(days)
+          Marty::Log.where(
+            'timestamp < ?', Time.zone.today - days.to_i.days
+          ).delete_all
+        end
+      end
+    end
+  end
+end

data/app/services/marty/cleaner/maintenance_window.rb

@@ -0,0 +1,49 @@
+module Marty
+  module Cleaner
+    module MaintenanceWindow
+      CONFIG_KEY = 'CLEANER_MAINTENANCE_WINDOW'
+      HASH_KEYS = ['day', 'range'].freeze
+      DEFAULT_DAY = 'saturday'
+      DEFAULT_RANGE = ['00:00', '24:00'].freeze
+      DAYNAMES = Date::DAYNAMES.map(&:downcase).to_set
+
+      class << self
+        def call
+          window = ::Marty::Config.fetch(CONFIG_KEY)
+          raise "'#{CONFIG_KEY}' is not a hash'" unless window.is_a?(Hash)
+
+          day, range = HASH_KEYS.map do |k|
+            raise "'#{k}' is missing from '#{CONFIG_KEY}'" unless window.key?(k)
+
+            window[k] || 'Marty::Cleaner::MaintenanceWindow'\
+                        "::DEFAULT_#{k.upcase}".constantize
+          end
+
+          raise '\'day\' must be a String' unless day.is_a?(String)
+
+          pday = day.downcase
+          raise '\'day\' must be a valid day of the week' unless
+            DAYNAMES.member?(pday)
+
+          # DisableTriggers call can impact the system so we only want to
+          # clean on a specific maintenance days
+          unless Time.zone.now.send("#{pday}?")
+            raise "#{name.demodulize} can only be called on "\
+                  "#{pday.capitalize}"
+          end
+
+          raise '\'range\' must be an array of length 2' unless
+            range.is_a?(Array) && range.size == 2
+
+          prange = range.map { |r| Time.zone.parse(r) }
+          raise 'invalid range specified' unless prange.all?
+
+          raise "Current time not within maintenance window: #{prange}" unless
+            Time.zone.now.between?(prange.first, prange.second)
+
+          window.merge('range' => prange, 'day' => pday)
+        end
+      end
+    end
+  end
+end

data/app/services/marty/cleaner/mcfly_models.rb

@@ -0,0 +1,29 @@
+module Marty
+  module Cleaner
+    module McflyModels
+      LOG_MESSAGE_TYPE = 'mcfly_model_cleaner'
+
+      CLASSES_TO_CLEAN = [
+        # Add these to monkey.rb in the format
+        # ::YourApp::YourModelA,
+        # ::YourApp::YourModelB
+      ].freeze
+
+      class << self
+        def call(days_to_keep)
+          CLASSES_TO_CLEAN.each do |klass|
+            table_name = klass.table_name
+            # Need to disable the McFly triggers first
+            ::Marty::McflyHelper::DisableTriggers.call(table_name) do
+              ::Marty::Cleaner::CleanAll.log(LOG_MESSAGE_TYPE, table_name) do
+                klass.where(
+                  'obsoleted_dt <= ?', Time.zone.now - days_to_keep.days
+                ).delete_all
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/marty/cleaner/timestamp_models.rb

@@ -0,0 +1,26 @@
+module Marty
+  module Cleaner
+    module TimestampModels
+      LOG_MESSAGE_TYPE = 'timestamp_model_cleaner'
+
+      CLASSES_TO_CLEAN = [
+        # Add these to monkey.rb in the format
+        # ::YourApp::YourModelA,
+        # ::YourApp::YourModelB
+      ].freeze
+
+      class << self
+        def call(days_to_keep)
+          CLASSES_TO_CLEAN.each do |klass|
+            table_name = klass.table_name
+            ::Marty::Cleaner::CleanAll.log(LOG_MESSAGE_TYPE, table_name) do
+              klass.where(
+                'updated_at <= ?', Time.zone.now - days_to_keep.days
+              ).delete_all
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/marty/mcfly_helper/disable_triggers.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Marty
+  module McflyHelper
+    module DisableTriggers
+      class << self
+        def call(*tables)
+          conn = ActiveRecord::Base.connection
+          tables.each do |table_name|
+            conn.execute("ALTER TABLE #{table_name} DISABLE TRIGGER USER;")
+          end
+
+          yield
+        ensure
+          tables.each do |table_name|
+            conn.execute("ALTER TABLE #{table_name} ENABLE TRIGGER USER;")
+          end
+        end
+      end
+    end
+  end
+end

data/db/migrate/604_schedule_cleaner_job.rb

@@ -0,0 +1,22 @@
+class ScheduleCleanerJob < ActiveRecord::Migration[5.1]
+  def up
+    noon_on_saturdays_cron = '0 12 * * 6'
+
+    Marty::BackgroundJob::Schedule.reset_column_information
+
+    schedule = Marty::BackgroundJob::Schedule.new(
+      job_class: 'Marty::CleanerJob',
+      cron: noon_on_saturdays_cron,
+      state: 'off',
+      arguments: [],
+    )
+
+    schedule.save!
+  end
+
+  def down
+    Marty::BackgroundJob::Schedule.find_by(
+      job_class: 'Marty::CleanerJob'
+    )&.destroy
+  end
+end

data/lib/marty/diagnostic/version.rb

@@ -1,5 +1,11 @@
 module Marty::Diagnostic
   class Version < Base
+    def self.git_tag
+      git_tag = `cd #{Rails.root}; git describe --tags --always --abbrev=7;`.strip
+      git_datetime = `cd #{Rails.root}; git log -1 --format=%cd;`.strip
+      "#{git_tag} (#{git_datetime})"
+    end
+
     diagnostic_fn do
       begin
         submodules = `cd #{Rails.root}; git submodule`.split("\n").map do |s|
@@ -11,12 +17,11 @@ module Marty::Diagnostic
           }
         end.reduce(&:merge) || {}
 
-        git_tag = `cd #{Rails.root}; git describe --tags --always --abbrev=7;`.strip
-        git_datetime = `cd #{Rails.root}; git log -1 --format=%cd;`.strip
-        git = { 'Root Git' => "#{git_tag} (#{git_datetime})" }.merge(submodules)
+        git = { 'Root Git' => git_tag }.merge(submodules)
       rescue StandardError
         git = { 'Root Git' => error('Failed accessing git') }
       end
+
       rbv = "#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL} (#{RUBY_PLATFORM})"
       {
         'Marty'                   => Marty::VERSION,

data/lib/marty/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Marty
-  VERSION = '14.0.0'
+  VERSION = '14.3.0'
 end

data/package.json

@@ -1,10 +1,10 @@
 {
   "private": true,
   "scripts": {
-    "eslint-check": "eslint  'app/**/*.{js,jsx}' ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
-    "eslint-write": "eslint --fix 'app/**/*.{js,jsx}' ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
-    "prettier-check": "prettier --check \"app/**/*.{js,jsx,css,scss}\" ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
-    "prettier-write": "prettier --write \"app/**/*.{js,jsx,css,scss}\" ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
+    "eslint-check": "eslint  'app/**/*.{js,jsx}' ./.schemalint-rules ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
+    "eslint-write": "eslint --fix 'app/**/*.{js,jsx}' ./.schemalint-rules ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
+    "prettier-check": "prettier --check \"app/**/*.{js,jsx,css,scss}\" ./.schemalint-rules/**/* ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
+    "prettier-write": "prettier --write \"app/**/*.{js,jsx,css,scss}\" ./.schemalint-rules/**/* ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
     "lint": "yarn run eslint-check && yarn run prettier-check",
     "lint-fix": "yarn run eslint-write && yarn run prettier-write",
     "lint-schema": "schemalint"

data/spec/controllers/diagnostic/controller_spec.rb

@@ -14,7 +14,7 @@ module Marty::Diagnostic
     end
 
     def git
-      tag = `cd #{Rails.root}; git describe --tags --always;`.strip
+      tag = `cd #{Rails.root}; git describe --tags --always --abbrev=7;`.strip
       git_datetime = `cd #{Rails.root}; git log -1 --format=%cd;`.strip
 
       "#{tag} (#{git_datetime})"

data/spec/dummy/config/initializers/delayed_job.rb

@@ -1,4 +1,4 @@
-ENV['DELAYED_VER'] = `cd #{Rails.root.to_s}; git describe --tags --always`.strip
+ENV['DELAYED_VER'] = Marty::Diagnostic::Version.git_tag
 
 if Rails.env.test?
   # set a really small delay in test environment so that it doesn't