marty 14.0.0 → 14.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.eslintignore +1 -0
- data/.eslintrc.js +2 -1
- data/.schemalint-rules/index.js +5 -0
- data/.schemalint-rules/preferVarcharWithoutSizeLimit.js +29 -0
- data/.schemalintrc.js +3 -6
- data/CHANGELOG.md +28 -0
- data/app/jobs/marty/cleaner_job.rb +16 -0
- data/app/models/marty/data_grid.rb +5 -3
- data/app/services/marty/cleaner/clean_all.rb +39 -0
- data/app/services/marty/cleaner/logs.rb +13 -0
- data/app/services/marty/cleaner/maintenance_window.rb +49 -0
- data/app/services/marty/cleaner/mcfly_models.rb +29 -0
- data/app/services/marty/cleaner/timestamp_models.rb +26 -0
- data/app/services/marty/mcfly_helper/disable_triggers.rb +22 -0
- data/db/migrate/604_schedule_cleaner_job.rb +22 -0
- data/lib/marty/diagnostic/version.rb +8 -3
- data/lib/marty/version.rb +1 -1
- data/package.json +4 -4
- data/spec/controllers/diagnostic/controller_spec.rb +1 -1
- data/spec/dummy/config/initializers/delayed_job.rb +1 -1
- data/spec/jobs/cleaner_job_spec.rb +16 -0
- data/spec/models/data_grid_spec.rb +9 -9
- data/spec/other/diagnostic/reporter_spec.rb +1 -1
- data/spec/services/cleaner/cleaner_spec.rb +139 -0
- data/yarn.lock +1093 -13
- metadata +14 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fd079baae110bb7b7b5fdfefa0cb1c5a1c03779cbab01c8ff3f92e587cb1e6e3
|
4
|
+
data.tar.gz: fdf66588c81dee9eec99458c29ea3a519cb454b40340ba344a3daab42da2544d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ad18ce255689328fbdb69490b4b47fbcdd97866953e92d7a68e838e2185443116539fbd65c16cc84c30ed9d9d08f2c8399b054cdced48769f48d414f523f5463
|
7
|
+
data.tar.gz: 1dd8b6f97729917f3e6dece1c20464eb01f3617fd6c5862cde4d897714680aee950bc255c4287f6b2c90183690d156978d11eefa2e1a2b86fda71a177efbf3b8
|
data/.eslintignore
CHANGED
data/.eslintrc.js
CHANGED
@@ -0,0 +1,29 @@
|
|
1
|
+
const preferVarcharWithoutSizeLimit = {
|
2
|
+
name: "prefer-varchar-without-size-limit",
|
3
|
+
docs: {
|
4
|
+
description: "Prefer using VARCHAR without size limit over varchar(255)",
|
5
|
+
url: "..."
|
6
|
+
},
|
7
|
+
|
8
|
+
process({ schemaObject, report }) {
|
9
|
+
const validator = ({ name: tableName }) => (column) => {
|
10
|
+
const columnName = column.name;
|
11
|
+
const type = column.type;
|
12
|
+
|
13
|
+
if (type.startsWith("varchar") && column.maxLength) {
|
14
|
+
report({
|
15
|
+
rule: this.name,
|
16
|
+
identifier: `${schemaObject.name}.${tableName}.${columnName}`,
|
17
|
+
message: `Prefer varchar to ${type}(${column.maxLength}) types`,
|
18
|
+
suggestedMigration: `ALTER TABLE "${tableName}" ALTER COLUMN "${columnName}" TYPE VARCHAR;`
|
19
|
+
});
|
20
|
+
}
|
21
|
+
};
|
22
|
+
|
23
|
+
schemaObject.tables.forEach((table) => {
|
24
|
+
table.columns.forEach(validator(table));
|
25
|
+
});
|
26
|
+
}
|
27
|
+
};
|
28
|
+
|
29
|
+
module.exports = preferVarcharWithoutSizeLimit;
|
data/.schemalintrc.js
CHANGED
@@ -7,16 +7,13 @@ module.exports = {
|
|
7
7
|
charset: "utf8"
|
8
8
|
},
|
9
9
|
|
10
|
-
|
10
|
+
plugins: ["./.schemalint-rules"],
|
11
11
|
|
12
12
|
rules: {
|
13
13
|
"name-casing": ["error", "snake"],
|
14
14
|
"name-inflection": ["error", "plural"],
|
15
|
-
"prefer-jsonb-to-json": ["error"]
|
16
|
-
|
17
|
-
// We would need to update lib so it would return column size info
|
18
|
-
// And create our own rule that checks that
|
19
|
-
// "prefer-text-to-varchar": ["error"]
|
15
|
+
"prefer-jsonb-to-json": ["error"],
|
16
|
+
"prefer-varchar-without-size-limit": ["error"]
|
20
17
|
},
|
21
18
|
|
22
19
|
schemas: [{ name: "public" }],
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,27 @@
|
|
1
|
+
14.2.0 - 2020-05-05
|
2
|
+
=====================================================
|
3
|
+
* Treating passed nil to data grids in the same way as missing attribute
|
4
|
+
broke our lookups. Roll that change back. With this change, the behavior
|
5
|
+
would be the following:
|
6
|
+
|
7
|
+
In non strict null mode:
|
8
|
+
missing attribute matches everything
|
9
|
+
passed nil matches only wildcard keys (empty keys)
|
10
|
+
|
11
|
+
In strict_null_mode:
|
12
|
+
missing attribute matches only NULLs and wildcard keys
|
13
|
+
passed nil matches only NULLs and wildcard keys
|
14
|
+
|
15
|
+
14.2.0 - 2020-05-05
|
16
|
+
=====================================================
|
17
|
+
* Adds `Marty::Diagnostic::Version.git_tag` method that is used in diags and can be redifined in Marty apps.
|
18
|
+
|
19
|
+
14.1.0 - 2020-05-01
|
20
|
+
=====================
|
21
|
+
* Add cleaner service and migration. The job is disabled by default.
|
22
|
+
|
23
|
+
In monkey.rb override `CLASSES_TO_CLEAN` with the desired classes to be scanned/cleaned.
|
24
|
+
|
1
25
|
14.0.0 - 2020-04-28
|
2
26
|
=====================================================
|
3
27
|
* Adds NULL support for data grid matchers:
|
@@ -9,6 +33,10 @@ NULL can be combined with other values in array
|
|
9
33
|
|
10
34
|
* DataGrid's PLPGSQL lookups are no longer supported
|
11
35
|
|
36
|
+
13.2.0 - 2020-04-30
|
37
|
+
=====================
|
38
|
+
* Add PDF content type handling
|
39
|
+
|
12
40
|
13.1.0 - 2020-04-14
|
13
41
|
=====================
|
14
42
|
* Use ruby for Marty::DataGrid lookups. That gives a small performance boost and simplifies the code.
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Marty
|
2
|
+
class CleanerJob < ::Marty::CronJob
|
3
|
+
def perform
|
4
|
+
system_login = Rails.configuration.marty.system_account
|
5
|
+
Marty::Logger.info("Starting CleanerJob as user: #{system_login}")
|
6
|
+
system_user = Marty::User.find_by(login: system_login)
|
7
|
+
|
8
|
+
Marty::Promises::Ruby::Create.call(
|
9
|
+
module_name: 'Marty::Cleaner::CleanAll',
|
10
|
+
method_name: 'call',
|
11
|
+
method_args: [],
|
12
|
+
params: { _user_id: system_user&.id }
|
13
|
+
)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -201,13 +201,15 @@ class Marty::DataGrid < Marty::Base
|
|
201
201
|
|
202
202
|
unless dgh['strict_null_mode']
|
203
203
|
next unless h_passed.key?(attr)
|
204
|
-
|
204
|
+
|
205
|
+
# FIXME: Treating passed nil in the same way
|
206
|
+
# as missing broke our lookups. Maybe we should get back to it later.
|
205
207
|
# Before missing attribute would match anything,
|
206
208
|
# while explicitly passed nil would only match wildcard keys
|
207
209
|
# We want to be consistent and treat nil attribute as missing one,
|
208
210
|
# unless it's a stict_null_mode, where nil would be explicitly mapped
|
209
211
|
# to NULL keys
|
210
|
-
next if val.nil?
|
212
|
+
# next if val.nil?
|
211
213
|
end
|
212
214
|
|
213
215
|
converted_val = if val.nil?
|
@@ -240,7 +242,7 @@ class Marty::DataGrid < Marty::Base
|
|
240
242
|
checks.all? do |check|
|
241
243
|
converted_val.send(check[0], check[1])
|
242
244
|
end
|
243
|
-
elsif key_val.nil?
|
245
|
+
elsif key_val.nil?
|
244
246
|
val.nil?
|
245
247
|
elsif m_type == 'boolean'
|
246
248
|
key_val == converted_val
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Marty
|
2
|
+
module Cleaner
|
3
|
+
module CleanAll
|
4
|
+
LOG_DAYS_KEY = 'log_days'
|
5
|
+
MCFLY_DAYS_KEY = 'mcfly_days'
|
6
|
+
TS_DAYS_KEY = 'timestamp_days'
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def log(table_type, table_name)
|
10
|
+
::Marty::Logger.log(table_type, 'Start Clean', table_name)
|
11
|
+
count = yield
|
12
|
+
::Marty::Logger.log(
|
13
|
+
table_type,
|
14
|
+
'End Clean',
|
15
|
+
"#{table_name} (#{count} records deleted)"
|
16
|
+
)
|
17
|
+
end
|
18
|
+
|
19
|
+
def call
|
20
|
+
window_config = ::Marty::Cleaner::MaintenanceWindow.call
|
21
|
+
log_days = window_config.fetch(LOG_DAYS_KEY, 60)
|
22
|
+
mcfly_days = window_config.fetch(MCFLY_DAYS_KEY, 365 * 3)
|
23
|
+
ts_days = window_config.fetch(TS_DAYS_KEY, 365 * 3)
|
24
|
+
|
25
|
+
[
|
26
|
+
[LOG_DAYS_KEY, log_days],
|
27
|
+
[MCFLY_DAYS_KEY, mcfly_days],
|
28
|
+
[TS_DAYS_KEY, ts_days]
|
29
|
+
].each do |key, value|
|
30
|
+
raise "'#{key}' must be an integer" unless value.is_a?(Integer)
|
31
|
+
end
|
32
|
+
::Marty::Cleaner::Logs.call(log_days)
|
33
|
+
::Marty::Cleaner::McflyModels.call(mcfly_days)
|
34
|
+
::Marty::Cleaner::TimestampModels.call(ts_days)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module Marty
|
2
|
+
module Cleaner
|
3
|
+
module MaintenanceWindow
|
4
|
+
CONFIG_KEY = 'CLEANER_MAINTENANCE_WINDOW'
|
5
|
+
HASH_KEYS = ['day', 'range'].freeze
|
6
|
+
DEFAULT_DAY = 'saturday'
|
7
|
+
DEFAULT_RANGE = ['00:00', '24:00'].freeze
|
8
|
+
DAYNAMES = Date::DAYNAMES.map(&:downcase).to_set
|
9
|
+
|
10
|
+
class << self
|
11
|
+
def call
|
12
|
+
window = ::Marty::Config.fetch(CONFIG_KEY)
|
13
|
+
raise "'#{CONFIG_KEY}' is not a hash'" unless window.is_a?(Hash)
|
14
|
+
|
15
|
+
day, range = HASH_KEYS.map do |k|
|
16
|
+
raise "'#{k}' is missing from '#{CONFIG_KEY}'" unless window.key?(k)
|
17
|
+
|
18
|
+
window[k] || 'Marty::Cleaner::MaintenanceWindow'\
|
19
|
+
"::DEFAULT_#{k.upcase}".constantize
|
20
|
+
end
|
21
|
+
|
22
|
+
raise '\'day\' must be a String' unless day.is_a?(String)
|
23
|
+
|
24
|
+
pday = day.downcase
|
25
|
+
raise '\'day\' must be a valid day of the week' unless
|
26
|
+
DAYNAMES.member?(pday)
|
27
|
+
|
28
|
+
# DisableTriggers call can impact the system so we only want to
|
29
|
+
# clean on a specific maintenance days
|
30
|
+
unless Time.zone.now.send("#{pday}?")
|
31
|
+
raise "#{name.demodulize} can only be called on "\
|
32
|
+
"#{pday.capitalize}"
|
33
|
+
end
|
34
|
+
|
35
|
+
raise '\'range\' must be an array of length 2' unless
|
36
|
+
range.is_a?(Array) && range.size == 2
|
37
|
+
|
38
|
+
prange = range.map { |r| Time.zone.parse(r) }
|
39
|
+
raise 'invalid range specified' unless prange.all?
|
40
|
+
|
41
|
+
raise "Current time not within maintenance window: #{prange}" unless
|
42
|
+
Time.zone.now.between?(prange.first, prange.second)
|
43
|
+
|
44
|
+
window.merge('range' => prange, 'day' => pday)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Marty
|
2
|
+
module Cleaner
|
3
|
+
module McflyModels
|
4
|
+
LOG_MESSAGE_TYPE = 'mcfly_model_cleaner'
|
5
|
+
|
6
|
+
CLASSES_TO_CLEAN = [
|
7
|
+
# Add these to monkey.rb in the format
|
8
|
+
# ::YourApp::YourModelA,
|
9
|
+
# ::YourApp::YourModelB
|
10
|
+
].freeze
|
11
|
+
|
12
|
+
class << self
|
13
|
+
def call(days_to_keep)
|
14
|
+
CLASSES_TO_CLEAN.each do |klass|
|
15
|
+
table_name = klass.table_name
|
16
|
+
# Need to disable the McFly triggers first
|
17
|
+
::Marty::McflyHelper::DisableTriggers.call(table_name) do
|
18
|
+
::Marty::Cleaner::CleanAll.log(LOG_MESSAGE_TYPE, table_name) do
|
19
|
+
klass.where(
|
20
|
+
'obsoleted_dt <= ?', Time.zone.now - days_to_keep.days
|
21
|
+
).delete_all
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Marty
|
2
|
+
module Cleaner
|
3
|
+
module TimestampModels
|
4
|
+
LOG_MESSAGE_TYPE = 'timestamp_model_cleaner'
|
5
|
+
|
6
|
+
CLASSES_TO_CLEAN = [
|
7
|
+
# Add these to monkey.rb in the format
|
8
|
+
# ::YourApp::YourModelA,
|
9
|
+
# ::YourApp::YourModelB
|
10
|
+
].freeze
|
11
|
+
|
12
|
+
class << self
|
13
|
+
def call(days_to_keep)
|
14
|
+
CLASSES_TO_CLEAN.each do |klass|
|
15
|
+
table_name = klass.table_name
|
16
|
+
::Marty::Cleaner::CleanAll.log(LOG_MESSAGE_TYPE, table_name) do
|
17
|
+
klass.where(
|
18
|
+
'updated_at <= ?', Time.zone.now - days_to_keep.days
|
19
|
+
).delete_all
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Marty
|
4
|
+
module McflyHelper
|
5
|
+
module DisableTriggers
|
6
|
+
class << self
|
7
|
+
def call(*tables)
|
8
|
+
conn = ActiveRecord::Base.connection
|
9
|
+
tables.each do |table_name|
|
10
|
+
conn.execute("ALTER TABLE #{table_name} DISABLE TRIGGER USER;")
|
11
|
+
end
|
12
|
+
|
13
|
+
yield
|
14
|
+
ensure
|
15
|
+
tables.each do |table_name|
|
16
|
+
conn.execute("ALTER TABLE #{table_name} ENABLE TRIGGER USER;")
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
class ScheduleCleanerJob < ActiveRecord::Migration[5.1]
|
2
|
+
def up
|
3
|
+
noon_on_saturdays_cron = '0 12 * * 6'
|
4
|
+
|
5
|
+
Marty::BackgroundJob::Schedule.reset_column_information
|
6
|
+
|
7
|
+
schedule = Marty::BackgroundJob::Schedule.new(
|
8
|
+
job_class: 'Marty::CleanerJob',
|
9
|
+
cron: noon_on_saturdays_cron,
|
10
|
+
state: 'off',
|
11
|
+
arguments: [],
|
12
|
+
)
|
13
|
+
|
14
|
+
schedule.save!
|
15
|
+
end
|
16
|
+
|
17
|
+
def down
|
18
|
+
Marty::BackgroundJob::Schedule.find_by(
|
19
|
+
job_class: 'Marty::CleanerJob'
|
20
|
+
)&.destroy
|
21
|
+
end
|
22
|
+
end
|
@@ -1,5 +1,11 @@
|
|
1
1
|
module Marty::Diagnostic
|
2
2
|
class Version < Base
|
3
|
+
def self.git_tag
|
4
|
+
git_tag = `cd #{Rails.root}; git describe --tags --always --abbrev=7;`.strip
|
5
|
+
git_datetime = `cd #{Rails.root}; git log -1 --format=%cd;`.strip
|
6
|
+
"#{git_tag} (#{git_datetime})"
|
7
|
+
end
|
8
|
+
|
3
9
|
diagnostic_fn do
|
4
10
|
begin
|
5
11
|
submodules = `cd #{Rails.root}; git submodule`.split("\n").map do |s|
|
@@ -11,12 +17,11 @@ module Marty::Diagnostic
|
|
11
17
|
}
|
12
18
|
end.reduce(&:merge) || {}
|
13
19
|
|
14
|
-
|
15
|
-
git_datetime = `cd #{Rails.root}; git log -1 --format=%cd;`.strip
|
16
|
-
git = { 'Root Git' => "#{git_tag} (#{git_datetime})" }.merge(submodules)
|
20
|
+
git = { 'Root Git' => git_tag }.merge(submodules)
|
17
21
|
rescue StandardError
|
18
22
|
git = { 'Root Git' => error('Failed accessing git') }
|
19
23
|
end
|
24
|
+
|
20
25
|
rbv = "#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL} (#{RUBY_PLATFORM})"
|
21
26
|
{
|
22
27
|
'Marty' => Marty::VERSION,
|
data/lib/marty/version.rb
CHANGED
data/package.json
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
{
|
2
2
|
"private": true,
|
3
3
|
"scripts": {
|
4
|
-
"eslint-check": "eslint 'app/**/*.{js,jsx}' ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
|
5
|
-
"eslint-write": "eslint --fix 'app/**/*.{js,jsx}' ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
|
6
|
-
"prettier-check": "prettier --check \"app/**/*.{js,jsx,css,scss}\" ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
|
7
|
-
"prettier-write": "prettier --write \"app/**/*.{js,jsx,css,scss}\" ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
|
4
|
+
"eslint-check": "eslint 'app/**/*.{js,jsx}' ./.schemalint-rules ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
|
5
|
+
"eslint-write": "eslint --fix 'app/**/*.{js,jsx}' ./.schemalint-rules ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
|
6
|
+
"prettier-check": "prettier --check \"app/**/*.{js,jsx,css,scss}\" ./.schemalint-rules/**/* ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
|
7
|
+
"prettier-write": "prettier --write \"app/**/*.{js,jsx,css,scss}\" ./.schemalint-rules/**/* ./.schemalintrc.js ./.eslintrc.js ./prettier.config.js",
|
8
8
|
"lint": "yarn run eslint-check && yarn run prettier-check",
|
9
9
|
"lint-fix": "yarn run eslint-write && yarn run prettier-write",
|
10
10
|
"lint-schema": "schemalint"
|
@@ -14,7 +14,7 @@ module Marty::Diagnostic
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def git
|
17
|
-
tag = `cd #{Rails.root}; git describe --tags --always;`.strip
|
17
|
+
tag = `cd #{Rails.root}; git describe --tags --always --abbrev=7;`.strip
|
18
18
|
git_datetime = `cd #{Rails.root}; git log -1 --format=%cd;`.strip
|
19
19
|
|
20
20
|
"#{tag} (#{git_datetime})"
|