marty 4.0.0.rc2 → 5.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +7 -0
  3. data/.rubocop.yml +1 -0
  4. data/.rubocop_todo.yml +3 -16
  5. data/.ssh-docker/.keep +0 -0
  6. data/Dockerfile.dummy +3 -0
  7. data/Gemfile +19 -15
  8. data/app/components/marty/base_rule_view.rb +104 -10
  9. data/app/components/marty/base_rule_view/client/base_rule_view.js +24 -0
  10. data/app/components/marty/data_grid_user_view.rb +39 -0
  11. data/app/components/marty/data_grid_view.rb +68 -18
  12. data/app/components/marty/extras/layout.rb +1 -1
  13. data/app/components/marty/grid.rb +1 -1
  14. data/app/components/marty/grid/client/grid.js +29 -13
  15. data/app/components/marty/import_view.rb +3 -3
  16. data/app/components/marty/main_auth_app.rb +11 -1
  17. data/app/components/marty/report_form.rb +6 -6
  18. data/app/components/marty/script_form.rb +5 -5
  19. data/app/components/marty/script_tester.rb +2 -2
  20. data/app/components/marty/user_view.rb +3 -9
  21. data/app/models/marty/base_rule.rb +92 -32
  22. data/app/models/marty/data_grid.rb +92 -22
  23. data/app/models/marty/event.rb +2 -2
  24. data/app/models/marty/promise.rb +4 -4
  25. data/app/models/marty/role_type.rb +14 -1
  26. data/app/services/marty/data_grid_view/save_grid.rb +2 -2
  27. data/app/services/marty/promises/delorean/create.rb +2 -2
  28. data/app/services/marty/promises/ruby/create.rb +2 -2
  29. data/config/locales/en.yml +11 -2
  30. data/db/migrate/108_add_data_grid_perms.rb +16 -0
  31. data/db/migrate/508_add_not_to_data_grids_tables.rb +18 -0
  32. data/db/migrate/509_update_dg_plpgsql_v1_fns.rb +13 -0
  33. data/db/sql/query_grid_dir_v1.sql +16 -2
  34. data/docker-compose.dummy.yml +1 -0
  35. data/lib/marty/content_handler.rb +2 -2
  36. data/lib/marty/data_change.rb +1 -1
  37. data/lib/marty/data_conversion.rb +3 -4
  38. data/lib/marty/data_importer.rb +4 -4
  39. data/lib/marty/mcfly_model.rb +7 -10
  40. data/lib/marty/migrations.rb +1 -1
  41. data/lib/marty/monkey.rb +2 -2
  42. data/lib/marty/promise_job.rb +5 -5
  43. data/lib/marty/promise_proxy.rb +2 -2
  44. data/lib/marty/promise_ruby_job.rb +4 -4
  45. data/lib/marty/version.rb +1 -1
  46. data/make-app.mk +1 -1
  47. data/marty.gemspec +13 -18
  48. data/other/marty/diagnostic/aws/ec2_instance.rb +17 -2
  49. data/other/marty/diagnostic/aws/error.rb +8 -0
  50. data/other/marty/diagnostic/database.rb +2 -2
  51. data/other/marty/diagnostic/delayed_job_version.rb +0 -1
  52. data/spec/dummy/app/components/gemini/my_rule_view.rb +32 -6
  53. data/spec/dummy/app/models/gemini/fannie_bup.rb +13 -20
  54. data/spec/dummy/app/models/gemini/my_rule.rb +4 -0
  55. data/spec/dummy/app/models/gemini/xyz_rule.rb +3 -1
  56. data/spec/dummy/config/application.rb +1 -0
  57. data/spec/dummy/config/initializers/secret_token.rb +1 -1
  58. data/spec/dummy/db/migrate/20190702115241_add_simple_guards_options_to_rules.rb +37 -0
  59. data/spec/features/data_grid_spec.rb +109 -47
  60. data/spec/features/reporting_spec.rb +4 -4
  61. data/spec/features/rule_spec.rb +62 -31
  62. data/spec/features/scripting_spec.rb +3 -3
  63. data/spec/features/user_view_spec.rb +17 -8
  64. data/spec/fixtures/csv/rule/MyRule.csv +4 -1
  65. data/spec/lib/data_importer_spec.rb +8 -8
  66. data/spec/lib/mcfly_model_spec.rb +6 -6
  67. data/spec/models/data_grid_spec.rb +139 -7
  68. data/spec/models/rule_spec.rb +116 -9
  69. data/spec/spec_helper.rb +2 -2
  70. data/spec/support/netzke.rb +4 -3
  71. metadata +55 -54
  72. data/Gemfile.lock +0 -289
@@ -311,7 +311,7 @@ SQL
311
311
 
312
312
  def self.cleanup
313
313
  where('start_dt < ?', Time.zone.now - 48.hours).delete_all
314
- rescue StandardError => exc
315
- Marty::Util.logger.error("event GC error: #{exc}")
314
+ rescue StandardError => e
315
+ Marty::Util.logger.error("event GC error: #{e}")
316
316
  end
317
317
  end
@@ -132,9 +132,9 @@ class Marty::Promise < Marty::Base
132
132
  # log "OFF0 #{Process.pid} #{last}"
133
133
  begin
134
134
  work_off_job(job)
135
- rescue StandardError => exc
135
+ rescue StandardError => e
136
136
  # log "OFFERR #{exc}"
137
- error = exception_to_result(exc)
137
+ error = exception_to_result(e)
138
138
  last.set_result(error)
139
139
  end
140
140
  # log "OFF1 #{Process.pid} #{last}"
@@ -215,8 +215,8 @@ class Marty::Promise < Marty::Base
215
215
  'start_dt < ? AND parent_id IS NULL',
216
216
  DateTime.now - (all ? 0.hours : 4.hours)
217
217
  ).destroy_all
218
- rescue StandardError => exc
219
- Marty::Util.logger.error("promise GC error: #{exc}")
218
+ rescue StandardError => e
219
+ Marty::Util.logger.error("promise GC error: #{e}")
220
220
  end
221
221
 
222
222
  def exception_to_result(promise:, exception:)
@@ -5,6 +5,19 @@ class Marty::RoleType < Marty::Base
5
5
  'admin',
6
6
  'user_manager',
7
7
  'dev',
8
- 'viewer'
8
+ 'viewer',
9
+ 'data_grid_editor'
9
10
  ]
11
+
12
+ def self.from_nice_names(roles)
13
+ Marty::RoleType.get_all.select do |role|
14
+ roles.include?(I18n.t("roles.#{role}", default: role))
15
+ end
16
+ end
17
+
18
+ def self.to_nice_names(roles)
19
+ roles.map do |role|
20
+ I18n.t("roles.#{role}", default: role)
21
+ end
22
+ end
10
23
  end
@@ -15,8 +15,9 @@ module Marty
15
15
  end
16
16
 
17
17
  def self.call(params)
18
- user_perm = Marty::DataGridView.get_edit_save_permission
19
18
  rec_id = params['record_id']
19
+ dg = Marty::DataGrid.mcfly_pt('infinity').find_by(group_id: rec_id)
20
+ user_perm = Marty::DataGridView.get_edit_permission(dg.permissions)
20
21
  data = params['data']
21
22
  raise GridError.new('entered with view permissions', data, rec_id) if
22
23
  user_perm == 'view'
@@ -24,7 +25,6 @@ module Marty
24
25
  data_as_array = data.map do |row|
25
26
  row.keys.map { |key| row[key] }
26
27
  end
27
- dg = Marty::DataGrid.mcfly_pt('infinity').find_by(group_id: rec_id)
28
28
  vcnt = dg.metadata.select { |md| md['dir'] == 'v' }.count
29
29
  hcnt = dg.metadata.select { |md| md['dir'] == 'h' }.count
30
30
  cur_data_dim = [dg.data.length, dg.data[0].length]
@@ -31,9 +31,9 @@ module Marty
31
31
  )
32
32
 
33
33
  job = Delayed::Job.enqueue(promise_job)
34
- rescue StandardError => exc
34
+ rescue StandardError => e
35
35
  # log "CALLERR #{exc}"
36
- res = ::Delorean::Engine.grok_runtime_exception(exc)
36
+ res = ::Delorean::Engine.grok_runtime_exception(e)
37
37
  promise.set_start
38
38
  promise.set_result(res)
39
39
  # log "CALLERRSET #{res}"
@@ -29,8 +29,8 @@ module Marty
29
29
  )
30
30
 
31
31
  job = Delayed::Job.enqueue(promise_job)
32
- rescue StandardError => exc
33
- res = { 'error' => exc.message }
32
+ rescue StandardError => e
33
+ res = { 'error' => e.message }
34
34
  promise.set_start
35
35
  promise.set_result(res)
36
36
  raise
@@ -18,8 +18,10 @@ en:
18
18
  create_posting: Create
19
19
  api_auth: API Authorization
20
20
  event_view: Event View
21
- data_grid_view: Data Grids
22
- data_grid: Data Grids
21
+ data_grid_view: Data Grids Admin
22
+ data_grid: Data Grids Admin
23
+ data_grid_user: Data Grids
24
+ data_grid_user_view: Data Grids
23
25
 
24
26
  data_import_view:
25
27
  import: Import
@@ -30,6 +32,7 @@ en:
30
32
  admin: Admin
31
33
  dev: Developer
32
34
  user_manager: User Manager
35
+ data_grid_editor: Data Grid Editor
33
36
 
34
37
  posting_types:
35
38
  BASE: BASE
@@ -176,3 +179,9 @@ en:
176
179
  grid:
177
180
  base:
178
181
  view_record: "%{model} (Read-only)"
182
+
183
+ data_grid_view_perms:
184
+ perm_view: Can View
185
+ perm_edit_data: Can Edit Data
186
+ perm_edit_all: Can Edit All
187
+
@@ -0,0 +1,16 @@
1
+ class AddDataGridPerms < ActiveRecord::Migration[4.2]
2
+ def self.up
3
+ default = {
4
+ view: [],
5
+ edit_data: [],
6
+ edit_all: []
7
+ }
8
+ table = :marty_data_grids
9
+ add_column table, :permissions, :jsonb, null: false, default: default
10
+ add_index table, "(permissions->'view')", using: "GIN",
11
+ name: :marty_data_grids_perm_idx
12
+ end
13
+ def self.down
14
+ remove_column :marty_data_grids, :permissions
15
+ end
16
+ end
@@ -0,0 +1,18 @@
1
+ class AddNotToDataGridsTables < ActiveRecord::Migration[5.1]
2
+ include Marty::Migrations
3
+
4
+ TABLES = {
5
+ marty_grid_index_booleans: :btree,
6
+ marty_grid_index_int4ranges: :gist,
7
+ marty_grid_index_integers: :gin,
8
+ marty_grid_index_numranges: :gist,
9
+ marty_grid_index_strings: :gin
10
+ }
11
+
12
+ def change
13
+ TABLES.each do |table, index_type|
14
+ add_column table, :not, :boolean, null: false, default: false
15
+ add_index table, [:not, :key]
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,13 @@
1
+ class UpdateDgPlpgsqlV1Fns < ActiveRecord::Migration[4.2]
2
+ def up
3
+ marty_path = Gem.loaded_specs["marty"].full_gem_path
4
+ Dir.glob("#{marty_path}/db/sql/*_v1.sql") do |f|
5
+ connection.execute(File.read(f))
6
+ end
7
+ end
8
+
9
+ def down
10
+ # Load functions once again
11
+ up
12
+ end
13
+ end
@@ -24,11 +24,17 @@ DECLARE
24
24
  sql_scripts_arr_intersect text;
25
25
  sql_filter text;
26
26
 
27
+ includes_nots boolean;
28
+
27
29
  BEGIN
28
30
  FOR i IN 1 .. COALESCE(array_upper(infos, 1), 0)
29
31
  LOOP
30
32
  attr_type := infos[i] ->> 'type';
31
33
  attr_name := infos[i] ->> 'attr';
34
+
35
+ -- Use not condition only if given type indexes has rows with 'not' = true
36
+ includes_nots = COALESCE(infos[i] -> 'nots', '[]'::JSONB) @> 'true';
37
+
32
38
  attr_value := h ->> attr_name;
33
39
  h_key_exists := h ? attr_name;
34
40
 
@@ -50,7 +56,7 @@ BEGIN
50
56
  END CASE;
51
57
 
52
58
  sql_script = 'SELECT DISTINCT index from ' || table_name ||
53
- -- Convertion to FLOAT is neeed to make numbers like 2005.0 work
59
+ -- Convertion to FLOAT is needed to make numbers like 2005.0 work
54
60
  ' WHERE data_grid_id = ($1 ->> ' || sqlidx || ')::FLOAT::INTEGER' ||
55
61
  ' AND created_dt = ($1 ->> ' || (sqlidx + 1) || ')::TIMESTAMP' ||
56
62
  ' AND attr = $1 ->> ' || (sqlidx + 2) || ' ';
@@ -83,7 +89,15 @@ BEGIN
83
89
  args := args || attr_value;
84
90
  END IF;
85
91
 
86
- sql_script := sql_script || ' AND (' || sql_filter || 'key is NULL) ';
92
+
93
+ -- Use not condition only if given type indexes has rows with 'not' = true
94
+ IF includes_nots THEN
95
+ sql_script := sql_script || ' AND CASE WHEN ' || table_name ||'.not '
96
+ 'THEN NOT (' || sql_filter || 'key IS NULL) '
97
+ 'ELSE (' || sql_filter || 'key IS NULL) END';
98
+ ELSE
99
+ sql_script := sql_script || ' AND (' || sql_filter || 'key is NULL) ';
100
+ END IF;
87
101
 
88
102
  sql_scripts_arr := sql_scripts_arr || sql_script;
89
103
  END LOOP;
@@ -24,6 +24,7 @@ services:
24
24
  - .:/opt/app:delegated
25
25
  - '.bash_history.docker:/root/.bash_history'
26
26
  - '.pry_history.docker:/root/.pry_history'
27
+ - '.ssh-docker:/root/.ssh'
27
28
  - bundle_box:/bundle_box
28
29
  tty: true
29
30
  stdin_open: true
@@ -34,9 +34,9 @@ module Marty::ContentHandler
34
34
  else
35
35
  res, format = { error: "Unknown format: #{format}" }.to_json, 'json'
36
36
  end
37
- rescue StandardError => exc
37
+ rescue StandardError => e
38
38
  res, format =
39
- { error: "Failed conversion #{format}: #{exc}" }.to_json, 'json'
39
+ { error: "Failed conversion #{format}: #{e}" }.to_json, 'json'
40
40
  end
41
41
 
42
42
  type, disposition = GEN_FORMATS[format]
@@ -219,7 +219,7 @@ class Marty::DataChange
219
219
  # "different".
220
220
  conv =
221
221
  Marty::DataConversion.convert_row(klass, input, ts)
222
- rescue StandardError => exc
222
+ rescue StandardError => e
223
223
  only_input << input
224
224
  next
225
225
  end
@@ -1,7 +1,7 @@
1
1
  class Marty::DataConversion
2
2
  EXCEL_START_DATE = Date.parse('1/1/1900') - 2
3
3
 
4
- FLOAT_PAT = /^-?\d+(\.\d+)?$/
4
+ FLOAT_PAT = /\A-?\d+(\.?\d+)?([eE][-+]?[0-9]+)?\z/
5
5
 
6
6
  PATS = {
7
7
  integer: /^-?\d+(\.0+)?$/,
@@ -58,13 +58,13 @@ class Marty::DataConversion
58
58
  begin
59
59
  v =~ FLOAT_PAT ? EXCEL_START_DATE + v.to_f :
60
60
  Mcfly.is_infinity(v) ? 'infinity' : v.to_date
61
- rescue StandardError => exc
61
+ rescue StandardError => e
62
62
  raise "date conversion failed for #{v.inspect}}"
63
63
  end
64
64
  when :datetime
65
65
  begin
66
66
  Mcfly.is_infinity(v) ? 'infinity' : v.to_datetime
67
- rescue StandardError => exc
67
+ rescue StandardError => e
68
68
  raise "datetime conversion failed for #{v.inspect}}"
69
69
  end
70
70
  when :numrange, :int4range, :int8range
@@ -98,7 +98,6 @@ class Marty::DataConversion
98
98
 
99
99
  @@associations[klass] ||= klass.reflect_on_all_associations.
100
100
  each_with_object({}) do |assoc, h|
101
-
102
101
  h[assoc.name.to_s] = {
103
102
  assoc_keys: assoc_keys(assoc.klass),
104
103
  assoc_class: assoc.klass,
@@ -77,9 +77,9 @@ module Marty
77
77
 
78
78
  Marty::DataConversion.create_or_update(klass, row, dt)
79
79
  end
80
- rescue StandardError => exc
80
+ rescue StandardError => e
81
81
  # to find problems with the importer, comment out the rescue block
82
- raise Error.new(exc.to_s, [eline])
82
+ raise Error.new(e.to_s, [eline])
83
83
  end
84
84
 
85
85
  ids = {}
@@ -96,8 +96,8 @@ module Marty
96
96
  # Validate affected rows if necessary
97
97
  klass.send(validation_function.to_sym, ids.keys) if
98
98
  validation_function
99
- rescue StandardError => exc
100
- raise Error.new(exc.to_s, [])
99
+ rescue StandardError => e
100
+ raise Error.new(e.to_s, [])
101
101
  end
102
102
 
103
103
  remainder_ids = cleaner_ids - ids.keys
@@ -6,10 +6,8 @@ module Mcfly::Model
6
6
  end
7
7
 
8
8
  module ClassMethods
9
- def hash_if_necessary(q, to_hash)
10
- return make_hash(q) if to_hash && q.is_a?(ActiveRecord::Base)
11
-
12
- q
9
+ def hash_if_necessary(q, private)
10
+ !private && q.is_a?(ActiveRecord::Base) ? make_openstruct(q) : q
13
11
  end
14
12
 
15
13
  def base_mcfly_lookup(name, options = {}, &block)
@@ -43,7 +41,7 @@ module Mcfly::Model
43
41
 
44
42
  q = q.first if q.respond_to?(:first) && options[:mode] == :first
45
43
 
46
- hash_if_necessary(q, options.fetch(:to_hash, false))
44
+ hash_if_necessary(q, options[:private])
47
45
  end
48
46
  end
49
47
 
@@ -57,7 +55,7 @@ module Mcfly::Model
57
55
 
58
56
  def gen_mcfly_lookup(name, attrs, options = {})
59
57
  raise "bad options #{options.keys}" unless
60
- (options.keys - [:mode, :cache, :private, :to_hash]).empty?
58
+ (options.keys - [:mode, :cache, :private]).empty?
61
59
 
62
60
  mode = options.fetch(:mode, :first)
63
61
 
@@ -88,7 +86,6 @@ module Mcfly::Model
88
86
 
89
87
  base_mcfly_lookup(name, options + { sig: attrs.length + 1,
90
88
  mode: mode }) do |_t, *attr_list|
91
-
92
89
  attr_list_ids = attr_list.each_with_index.map do |_x, i|
93
90
  assoc.member?(attrs[i]) ?
94
91
  (attr_list[i] && attr_list[i].id) : attr_list[i]
@@ -137,7 +134,7 @@ module Mcfly::Model
137
134
 
138
135
  pc_name = "pc_#{name}".to_sym
139
136
 
140
- gen_mcfly_lookup(pc_name, pc_attrs, options + { private: true, to_hash: false })
137
+ gen_mcfly_lookup(pc_name, pc_attrs, options + { private: true })
141
138
 
142
139
  lpi = attrs.keys.index rel_attr
143
140
 
@@ -145,7 +142,7 @@ module Mcfly::Model
145
142
  raise "need #{rel_attr} argument" unless lpi
146
143
 
147
144
  # cache if mode is not nil
148
- to_hash = options.fetch(:to_hash, false)
145
+ priv = options[:private]
149
146
 
150
147
  # cache if mode is not explicitly set to nil or cache is true
151
148
  cache = options.fetch(:cache) { options.fetch(:mode, :first) }
@@ -168,7 +165,7 @@ module Mcfly::Model
168
165
  send(cat_attr_id)
169
166
 
170
167
  q = send(pc_name, ts, *args)
171
- hash_if_necessary(q, to_hash)
168
+ hash_if_necessary(q, priv)
172
169
  end
173
170
  end
174
171
  end
@@ -41,7 +41,7 @@ module Marty::Migrations
41
41
  SQL
42
42
 
43
43
  db_values = res.first['enum_range'].gsub(/[{"}]/, '').split(',')
44
- ex_values = klass::VALUES - db_values
44
+ ex_values = klass::VALUES.map(&:to_s) - db_values
45
45
 
46
46
  return if ex_values.empty?
47
47
 
@@ -54,8 +54,8 @@ class Delorean::BaseModule::NodeCall
54
54
  begin
55
55
  # make sure params is serialzable before starting a Job
56
56
  JSON.dump(params)
57
- rescue StandardError => exc
58
- raise "non-serializable parameters: #{params} #{exc}"
57
+ rescue StandardError => e
58
+ raise "non-serializable parameters: #{params} #{e}"
59
59
  end
60
60
 
61
61
  Marty::Promises::Delorean::Create.call(
@@ -41,9 +41,9 @@ class Marty::PromiseJob < Struct.new(:promise,
41
41
  end
42
42
 
43
43
  # log "DONE #{Process.pid} #{promise.id} #{Time.now.to_f} #{res}"
44
- rescue StandardError => exc
45
- res = Delorean::Engine.grok_runtime_exception(exc)
46
- # log "ERR- #{Process.pid} #{promise.id} #{Time.now.to_f} #{exc}"
44
+ rescue StandardError => e
45
+ res = Delorean::Engine.grok_runtime_exception(e)
46
+ # log "ERR- #{Process.pid} #{promise.id} #{Time.now.to_f} #{e}"
47
47
  end
48
48
  promise.set_result(res)
49
49
  process_hook(res)
@@ -53,8 +53,8 @@ class Marty::PromiseJob < Struct.new(:promise,
53
53
  return unless hook
54
54
 
55
55
  hook.run(params: params, result: res)
56
- rescue StandardError => exc
57
- Marty::Util.logger.error "promise hook failed: #{exc}"
56
+ rescue StandardError => e
57
+ Marty::Util.logger.error "promise hook failed: #{e}"
58
58
  end
59
59
 
60
60
  def max_attempts
@@ -55,8 +55,8 @@ class Marty::PromiseProxy < BasicObject
55
55
  begin
56
56
  @result = @promise.wait_for_result(@timeout)
57
57
  @result = @result[@attr] if @attr && !@result['error']
58
- rescue ::Exception => exc
59
- @result = ::Delorean::Engine.grok_runtime_exception(exc)
58
+ rescue ::Exception => e
59
+ @result = ::Delorean::Engine.grok_runtime_exception(e)
60
60
  end
61
61
  end
62
62
  end
@@ -25,8 +25,8 @@ class Marty::PromiseRubyJob < Struct.new(:promise,
25
25
 
26
26
  mod = module_name.constantize
27
27
  res = { 'result' => mod.send(method_name, *method_args) }
28
- rescue StandardError => exc
29
- res = ::Marty::Promise.exception_to_result(promise: promise, exception: exc)
28
+ rescue StandardError => e
29
+ res = ::Marty::Promise.exception_to_result(promise: promise, exception: e)
30
30
  end
31
31
 
32
32
  promise.set_result(res)
@@ -37,8 +37,8 @@ class Marty::PromiseRubyJob < Struct.new(:promise,
37
37
  return unless hook
38
38
 
39
39
  hook.run(params: method_args, result: res)
40
- rescue StandardError => exc
41
- Marty::Util.logger.error "promise hook failed: #{exc}"
40
+ rescue StandardError => e
41
+ Marty::Util.logger.error "promise hook failed: #{e}"
42
42
  end
43
43
 
44
44
  def max_attempts