marty 1.0.54 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b39a9aaf4fd1b9f90bdff0503ea509ae7b4e4232
4
- data.tar.gz: 36f9283c373a1e5e1f0ab1c5e5a00efab731238c
3
+ metadata.gz: 5f65bd901e74662263479f7acb3c2f616e41be2a
4
+ data.tar.gz: 6ef39fe48e763f6758e93d0f75aae5266bb3ab64
5
5
  SHA512:
6
- metadata.gz: a0212491f9ac246e2731fc2629d7543f49f0c814b597459cef568be5fc824ce891b56553f083f5de3a831ff8cd6585e118c82379f2512d74f08eff4913f08b40
7
- data.tar.gz: abf2e69338e8f90b1f9dac8b15a31896d1ffab3681190adece922db454ac8dada1ecce411cef5c417f3d5d161d0a0e5f250bfa917de03e8ef17fe069163e9b77
6
+ metadata.gz: bf459ac2132cd88ce789485e564c1885b2433a8bf2fed54239249d660038db911dfec94345e8166ee279c8b1955aa7a93812fed51e16013084679e8235c2fd5d
7
+ data.tar.gz: 5c89ea1896bfed735b88a6cf3e0d8e507d7ef329d35c53cf1e933d1183826861fa411378f3dbf36dc6de6882a5bfc399b3e12803754b602aa0440b4e3c4c37c5
data/Gemfile CHANGED
@@ -26,7 +26,9 @@ group :development, :test do
26
26
  gem 'netzke-core'
27
27
  gem 'netzke-basepack'
28
28
  gem 'netzke-testing'
29
+ gem 'rspec-instafail', require: false
29
30
 
30
31
  gem 'marty_rspec'
31
- gem 'rspec-instafail', require: false
32
+
33
+ # gem 'delorean_lang', path: File.expand_path('../../delorean', __FILE__)
32
34
  end
@@ -1,10 +1,10 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- marty (1.0.54)
4
+ marty (1.1.0)
5
5
  axlsx (= 2.1.0pre)
6
6
  coderay
7
- delorean_lang (~> 0.1)
7
+ delorean_lang (~> 0.3.24)
8
8
  json-schema
9
9
  mcfly (= 0.0.19)
10
10
  net-ldap (~> 0.12.0)
@@ -1,4 +1,24 @@
1
1
  class Marty::Base < ActiveRecord::Base
2
2
  self.table_name_prefix = "marty_"
3
3
  self.abstract_class = true
4
+
5
+ def self.mcfly_pt(pt)
6
+ tb = self.table_name
7
+ self.where("#{tb}.obsoleted_dt >= ? AND #{tb}.created_dt < ?", pt, pt)
8
+ end
9
+ MCFLY_PT_SIG = [1, 1]
10
+
11
+ # FIXME: hacky signatures for AR queries
12
+ COUNT_SIG = [0, 0]
13
+ DISTINCT_SIG = [0, 100]
14
+ FIRST_SIG = [0, 1]
15
+ GROUP_SIG = [1, 100]
16
+ JOINS_SIG = [1, 100]
17
+ LAST_SIG = [0, 1]
18
+ LIMIT_SIG = [1, 1]
19
+ NOT_SIG = [1, 100]
20
+ ORDER_SIG = [1, 100]
21
+ PLUCK_SIG = [1, 100]
22
+ SELECT_SIG = [1, 100]
23
+ WHERE_SIG = [0, 100]
4
24
  end
@@ -86,11 +86,8 @@ class Marty::DataGrid < Marty::Base
86
86
  validates_with DataGridValidator
87
87
  validates_with Marty::NameValidator, field: :name
88
88
 
89
- gen_mcfly_lookup :lookup, {
90
- name: false,
91
- }
92
-
93
- gen_mcfly_lookup :get_all, {}, mode: :all
89
+ gen_mcfly_lookup :lookup, [:name], cache: true
90
+ gen_mcfly_lookup :get_all, [], mode: nil
94
91
 
95
92
  cached_mcfly_lookup :lookup_id, sig: 2 do
96
93
  |pt, group_id|
@@ -10,11 +10,8 @@ class Marty::Script < Marty::Base
10
10
 
11
11
  belongs_to :user, class_name: "Marty::User"
12
12
 
13
- gen_mcfly_lookup :lookup, {
14
- name: false,
15
- }
16
-
17
- gen_mcfly_lookup :get_all, {}, mode: :all
13
+ gen_mcfly_lookup :lookup, [:name], cache: true
14
+ gen_mcfly_lookup :get_all, {}, mode: nil
18
15
 
19
16
  # find script by name/tag
20
17
  def self.find_script(sname, tag=nil)
@@ -10,7 +10,7 @@
10
10
 
11
11
  require 'marty/engine'
12
12
  require 'marty/railtie'
13
- require 'marty/mcfly_query'
13
+ require 'marty/mcfly_model'
14
14
  require 'marty/monkey'
15
15
  require 'marty/promise_job'
16
16
  require 'marty/lazy_column_loader'
@@ -0,0 +1,188 @@
1
+ require 'mcfly'
2
+
3
+ module Mcfly::Model
4
+ def self.included(base)
5
+ base.send :extend, ClassMethods
6
+ end
7
+
8
+ module ClassMethods
9
+ def clear_lookup_cache!
10
+ @LOOKUP_CACHE.clear if @LOOKUP_CACHE
11
+ end
12
+
13
+ # FIXME IDEA: we just make :cache an argument to delorean_fn.
14
+ # That way, we don't need the cached_ flavors. It'll make all
15
+ # this code a lot simpler. We should also just add the :private
16
+ # mechanism here.
17
+
18
+ # Implements a VERY HACKY class-based (per process) caching
19
+ # mechanism for database lookup results. Issues include: cached
20
+ # values are ActiveRecord objects. Query results can be very
21
+ # large lists which we count as one item in the cache. Caching
22
+ # mechanism will result in large processes.
23
+ def cached_delorean_fn(name, options = {}, &block)
24
+ @LOOKUP_CACHE ||= {}
25
+
26
+ delorean_fn(name, options) do |ts, *args|
27
+ cache_key = [name, ts] + args.map{ |a|
28
+ a.is_a?(ActiveRecord::Base) ? a.id : a
29
+ } unless Mcfly.is_infinity(ts)
30
+
31
+ next @LOOKUP_CACHE[cache_key] if
32
+ cache_key && @LOOKUP_CACHE.has_key?(cache_key)
33
+
34
+ res = block.call(ts, *args)
35
+
36
+ if cache_key
37
+ # Cache has >1000 items, clear out the oldest 200. FIXME:
38
+ # hard-coded, should be configurable. Cache
39
+ # size/invalidation should be per lookup and not class.
40
+ # We're invalidating cache items simply based on age and
41
+ # not usage. This is faster but not as fair.
42
+ if @LOOKUP_CACHE.count > 1000
43
+ @LOOKUP_CACHE.keys[0..200].each{|k| @LOOKUP_CACHE.delete(k)}
44
+ end
45
+ @LOOKUP_CACHE[cache_key] = res
46
+
47
+ # Since we're caching this object and don't want anyone
48
+ # changing it. FIXME: ideally should freeze this object
49
+ # recursively.
50
+ res.freeze unless res.is_a?(ActiveRecord::Relation)
51
+ end
52
+ res
53
+ end
54
+ end
55
+
56
+ # FIXME: duplicate code from Mcfly's mcfly_lookup.
57
+ def cached_mcfly_lookup(name, options = {}, &block)
58
+ cached_delorean_fn(name, options) do |ts, *args|
59
+ raise "nil timestamp" if ts.nil?
60
+
61
+ ts = Mcfly.normalize_infinity(ts)
62
+
63
+ self.mcfly_pt(ts).scoping do
64
+ block.call(ts, *args)
65
+ end
66
+ end
67
+ end
68
+
69
+ # FIXME: add private mode. This should make the function
70
+ # unavailable to delorean.
71
+ def gen_mcfly_lookup(name, attrs, options={})
72
+ raise "bad options #{options.keys}" unless
73
+ (options.keys - [:mode, :cache, :private]).empty?
74
+
75
+ mode = options.fetch(:mode, :first)
76
+
77
+ # if mode is nil, don't cache -- i.e. don't cache AR queries
78
+ cache = mode && options[:cache]
79
+
80
+ # the older mode=:all is not supported (it's bogus)
81
+ raise "bad mode #{mode}" unless [nil, :first].member?(mode)
82
+
83
+ assoc = Set.new(self.reflect_on_all_associations.map(&:name))
84
+
85
+ qstr = attrs.map {|k, v|
86
+ k = "#{k}_id" if assoc.member?(k)
87
+
88
+ v ? "(#{k} = ? OR #{k} IS NULL)" : "(#{k} = ?)"
89
+ }.join(" AND ")
90
+
91
+ if Hash === attrs
92
+ order = attrs.select {|k, v| v}.keys.reverse.map { |k|
93
+ k = "#{k}_id" if assoc.member?(k)
94
+
95
+ "#{k} NULLS LAST"
96
+ }.join(", ")
97
+ attrs = attrs.keys
98
+ else
99
+ raise "bad attrs" unless Array === attrs
100
+ end
101
+
102
+ fn = cache ? :cached_mcfly_lookup : :mcfly_lookup
103
+
104
+ # hacky: if private, set sig to bad value -- i.e. can't be
105
+ # called from delorean. Ideally, we should have a 'private'
106
+ # option for delorean_fn.
107
+ sig = options[:private] ? -1 : attrs.length+1
108
+
109
+ send(fn, name, sig: sig) do
110
+ |t, *attr_list|
111
+
112
+ attr_list_ids = attr_list.each_with_index.map {|x, i|
113
+ assoc.member?(attrs[i]) ?
114
+ (attr_list[i] && attr_list[i].id) : attr_list[i]
115
+ }
116
+
117
+ q = self.where(qstr, *attr_list_ids)
118
+ q = q.order(order) if order
119
+ mode ? q.send(mode) : q
120
+ end
121
+ end
122
+
123
+ ######################################################################
124
+
125
+ # Generates categorization lookups, e.g. given class GFee:
126
+
127
+ # gen_mcfly_lookup_cat :lookup_q,
128
+ # [:security_instrument,
129
+ # 'Gemini::SecurityInstrumentCategorization',
130
+ # :g_fee_category],
131
+ # {
132
+ # entity: true,
133
+ # security_instrument: true,
134
+ # coupon: true,
135
+ # },
136
+ # nil
137
+
138
+ # rel_attr = :security_instrument
139
+ # cat_assoc_klass = Gemini::SecurityInstrumentCategorization
140
+ # cat_attr = :g_fee_category
141
+ # name = :lookup_q
142
+ # pc_name = :pc_lookup_q
143
+ # pc_attrs = {entity: true, security_instrument: true, coupon: true}
144
+
145
+ def gen_mcfly_lookup_cat(name, catrel, attrs, options={})
146
+ rel_attr, cat_assoc_name, cat_attr = catrel
147
+
148
+ raise "#{rel_attr} should be mapped in attrs" if attrs[rel_attr].nil?
149
+
150
+ cat_assoc_klass = cat_assoc_name.constantize
151
+
152
+ # replace rel_attr with cat_attr in attrs
153
+ pc_attrs = attrs.each_with_object({}) {|(k, v), h|
154
+ h[k == rel_attr ? "#{cat_attr}_id" : k] = v
155
+ }
156
+
157
+ pc_name = "pc_#{name}".to_sym
158
+
159
+ gen_mcfly_lookup(pc_name, pc_attrs, options + {private: true})
160
+
161
+ lpi = attrs.keys.index rel_attr
162
+
163
+ raise "should not include #{cat_attr}" if attrs.member?(cat_attr)
164
+ raise "need #{rel_attr} argument" unless lpi
165
+
166
+ # cache if mode is not nil
167
+ fn = options.fetch(:mode, :first) ? :cached_delorean_fn : :delorean_fn
168
+
169
+ send(fn, name, sig: attrs.length+1) do
170
+ |ts, *args|
171
+
172
+ # Example: rel is a Gemini::SecurityInstrument instance.
173
+ rel = args[lpi]
174
+ raise "#{rel_attr} can't be nil" unless rel
175
+
176
+ args[lpi] = cat_assoc_klass.
177
+ mcfly_pt(ts).
178
+ # FIXME: XXXX why is this join needed???
179
+ # joins(cat_attr).
180
+ where(rel_attr => rel).
181
+ pluck("#{cat_attr}_id").
182
+ first
183
+
184
+ self.send(pc_name, ts, *args)
185
+ end
186
+ end
187
+ end
188
+ end
@@ -202,7 +202,7 @@ end
202
202
  class StringEnum < String
203
203
  include Delorean::Model
204
204
  def name
205
- self
205
+ self.to_s
206
206
  end
207
207
  def id
208
208
  self
@@ -224,6 +224,7 @@ class StringEnum < String
224
224
  new(v)
225
225
  end
226
226
  end
227
+
227
228
  YAML::add_domain_type("pennymac.com,2017-06-02", "stringEnum") do
228
229
  |type, val|
229
230
  StringEnum.new(val)
@@ -252,3 +253,59 @@ module ActiveRecord
252
253
  end
253
254
  end
254
255
  end
256
+
257
+ ######################################################################
258
+
259
+ class ActiveRecord::Relation
260
+ def mcfly_pt(pt, cls=nil)
261
+ cls ||= self.klass
262
+ tb = cls.table_name
263
+ self.where("#{tb}.obsoleted_dt >= ? AND #{tb}.created_dt < ?", pt, pt)
264
+ end
265
+ end
266
+
267
+ ######################################################################
268
+
269
+ class ActiveRecord::Base
270
+ class << self
271
+ alias_method :old_joins, :joins
272
+
273
+ def joins(*args)
274
+ # when joins args are strings, checks to see if they're
275
+ # associations attrs. If so, convert them to symbols for joins
276
+ # to work properly.
277
+ new_args = args.map {|a|
278
+ self.reflections.has_key?(a) ? a.to_sym : a
279
+ }
280
+ old_joins(*new_args)
281
+ end
282
+ end
283
+ end
284
+
285
+ args_hack = [[ActiveRecord::Relation, ActiveRecord::QueryMethods::WhereChain]] +
286
+ [[Object, nil]]*10
287
+
288
+ Delorean::RUBY_WHITELIST.merge!(
289
+ count: [ActiveRecord::Relation],
290
+ distinct: args_hack,
291
+ group: args_hack,
292
+ joins: args_hack,
293
+ limit: [ActiveRecord::Relation, Integer],
294
+ not: args_hack,
295
+ order: args_hack,
296
+ pluck: args_hack,
297
+ select: args_hack,
298
+ where: args_hack,
299
+ mcfly_pt: [ActiveRecord::Relation,
300
+ [Date, Time, ActiveSupport::TimeWithZone, String],
301
+ [nil, Class]],
302
+ )
303
+
304
+ ######################################################################
305
+
306
+ module Mcfly::Controller
307
+ # define mcfly user to be Flowscape's current_user.
308
+ def user_for_mcfly
309
+ find_current_user rescue nil
310
+ end
311
+ end
@@ -1,3 +1,3 @@
1
1
  module Marty
2
- VERSION = "1.0.54"
2
+ VERSION = "1.1.1"
3
3
  end
@@ -34,7 +34,7 @@ Gem::Specification.new do |s|
34
34
 
35
35
  s.add_dependency 'axlsx', '2.1.0pre'
36
36
 
37
- s.add_dependency 'delorean_lang', '~> 0.1'
37
+ s.add_dependency 'delorean_lang', '~> 0.3.24'
38
38
  s.add_dependency 'mcfly', '0.0.19'
39
39
 
40
40
  s.add_dependency 'coderay'
@@ -1,5 +1,5 @@
1
1
  module Gemini
2
- class BudCategory < ActiveRecord::Base
2
+ class BudCategory < Marty::Base
3
3
  self.table_name = 'gemini_bud_categories'
4
4
  has_mcfly append_only: true
5
5
  mcfly_validates_uniqueness_of :name
@@ -1,5 +1,5 @@
1
1
  module Gemini
2
- class FannieBup < ActiveRecord::Base
2
+ class FannieBup < Marty::Base
3
3
  extend Gemini::Extras::DataImport
4
4
  extend Gemini::Extras::SettlementImport
5
5
 
@@ -2,4 +2,4 @@
2
2
  require File.expand_path('../application', __FILE__)
3
3
 
4
4
  # Initialize the rails application
5
- Dummy::Application.initialize!
5
+ Dummy::Application.initialize! unless Dummy::Application.initialized?
@@ -0,0 +1,159 @@
1
+ require "spec_helper"
2
+
3
+ module Marty
4
+
5
+ bud_cats =<<EOF
6
+ name
7
+ Conv Fixed 30
8
+ Conv Fixed 20
9
+ EOF
10
+
11
+ fannie_bup =<<EOF
12
+ bud_category note_rate buy_up buy_down settlement_mm settlement_yy
13
+ Conv Fixed 30 2.250 4.42000 7.24000 12 2012
14
+ Conv Fixed 30 2.375 4.42000 7.24000 12 2012
15
+ Conv Fixed 30 2.500 4.41300 7.22800 12 2012
16
+ Conv Fixed 30 2.625 4.37500 7.16200 12 2012
17
+ Conv Fixed 30 2.750 4.32900 7.09300 12 2012
18
+ Conv Fixed 20 2.875 4.24800 6.95900 12 2012
19
+ Conv Fixed 20 2.875 4.24800 6.95900 11 2012
20
+ EOF
21
+
22
+ script =<<EOF
23
+ A:
24
+ c = Gemini::FannieBup.
25
+ joins("bud_category").
26
+ where("name LIKE '%30'").
27
+ count
28
+
29
+ s = Gemini::FannieBup.
30
+ joins("bud_category").
31
+ select("name").
32
+ distinct("name").
33
+ pluck("name")
34
+
35
+ o = Gemini::FannieBup.
36
+ order("note_rate DESC", "buy_down ASC").
37
+ select("note_rate").
38
+ first.note_rate
39
+
40
+ g = Gemini::FannieBup.
41
+ select("settlement_yy*settlement_mm AS x, count(*) AS c").
42
+ group("settlement_mm", "settlement_yy").
43
+ order("settlement_mm").to_a
44
+
45
+ gg = [r.attributes for r in g]
46
+
47
+ n = Gemini::FannieBup.where.not("settlement_mm < 12").count
48
+
49
+ q = Gemini::FannieBup.where("settlement_mm = 12")
50
+ q1 = q.order("note_rate ASC").pluck("note_rate")
51
+ q2 = q.order("note_rate DESC").pluck("note_rate")
52
+
53
+ settlement_mm =?
54
+ note_rate =?
55
+ pq = Gemini::FannieBup.
56
+ where({"settlement_mm": settlement_mm}).
57
+ where("note_rate > ?", note_rate).
58
+ pluck("note_rate")
59
+
60
+ m = Gemini::FannieBup.
61
+ joins("bud_category").
62
+ mcfly_pt('infinity').
63
+ select("name").
64
+ pluck("name")
65
+
66
+ mm = Gemini::FannieBup.
67
+ mcfly_pt('01-01-2003').count
68
+ EOF
69
+
70
+ describe 'DeloreanQuery' do
71
+ before(:each) do
72
+ marty_whodunnit
73
+ Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
74
+ Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup)
75
+
76
+ Marty::Script.load_script_bodies(
77
+ {
78
+ "A" => script,
79
+ }, Date.today)
80
+
81
+ @engine = Marty::ScriptSet.new.get_engine("A")
82
+ end
83
+
84
+ it "perfroms join+count" do
85
+ res = @engine.evaluate("A", "c", {})
86
+
87
+ expect(res).to eq Gemini::FannieBup.
88
+ joins("bud_category").
89
+ where("name LIKE '%30'").
90
+ count
91
+
92
+ end
93
+
94
+ it "perfroms select+distinct" do
95
+ res = @engine.evaluate("A", "s", {})
96
+
97
+ expect(res).to eq Gemini::FannieBup.
98
+ joins("bud_category").
99
+ select("name").
100
+ distinct("name").
101
+ pluck("name")
102
+ end
103
+
104
+ it "perfroms mcfly_pt" do
105
+ res = @engine.evaluate("A", ["m", "mm"], {})
106
+
107
+ expect(res).to eq [
108
+ Gemini::FannieBup.
109
+ joins("bud_category").
110
+ mcfly_pt('infinity').
111
+ select("name").
112
+ pluck("name"),
113
+ Gemini::FannieBup.
114
+ mcfly_pt('01-01-2003').count,
115
+ ]
116
+ end
117
+
118
+ it "perfroms order+first" do
119
+ res = @engine.evaluate("A", "o", {})
120
+
121
+ expect(res).to eq Gemini::FannieBup.
122
+ order("note_rate DESC", "buy_down ASC").
123
+ select("note_rate").
124
+ first.note_rate
125
+ end
126
+
127
+ it "perfroms group+count" do
128
+ res = @engine.evaluate("A", "gg", {})
129
+
130
+ expect(res).
131
+ to eq Gemini::FannieBup.
132
+ select("settlement_yy*settlement_mm AS x, count(*) AS c").
133
+ group("settlement_mm", "settlement_yy").
134
+ order("settlement_mm").
135
+ map(&:attributes)
136
+ end
137
+
138
+ it "perfroms where+not" do
139
+ res = @engine.evaluate("A", "n", {})
140
+
141
+ expect(res).to eq Gemini::FannieBup.where.not("settlement_mm < 12").count
142
+ end
143
+
144
+ it "perfroms query+query" do
145
+ res = @engine.evaluate("A", ["q1", "q2"], {})
146
+
147
+ expect(res).to eq [
148
+ [2.25, 2.375, 2.5, 2.625, 2.75, 2.875],
149
+ [2.875, 2.75, 2.625, 2.5, 2.375, 2.25],
150
+ ]
151
+ end
152
+
153
+ it "handle query params" do
154
+ res = @engine.evaluate("A", "pq",
155
+ {"settlement_mm" => 12, "note_rate" => 2.5})
156
+ expect(res).to eq [2.625, 2.75, 2.875]
157
+ end
158
+ end
159
+ end
@@ -5,7 +5,7 @@ require 'rspec/rails'
5
5
  require 'database_cleaner'
6
6
  require 'marty_rspec'
7
7
 
8
- Dummy::Application.initialize!
8
+ Dummy::Application.initialize! unless Dummy::Application.initialized?
9
9
 
10
10
  ActiveRecord::Migrator.migrate File.expand_path("../../db/migrate/", __FILE__)
11
11
  ActiveRecord::Migrator.migrate File.expand_path("../dummy/db/migrate/", __FILE__)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: marty
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.54
4
+ version: 1.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Arman Bostani
@@ -14,7 +14,7 @@ authors:
14
14
  autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
- date: 2017-11-28 00:00:00.000000000 Z
17
+ date: 2017-12-04 00:00:00.000000000 Z
18
18
  dependencies:
19
19
  - !ruby/object:Gem::Dependency
20
20
  name: pg
@@ -92,14 +92,14 @@ dependencies:
92
92
  requirements:
93
93
  - - "~>"
94
94
  - !ruby/object:Gem::Version
95
- version: '0.1'
95
+ version: 0.3.24
96
96
  type: :runtime
97
97
  prerelease: false
98
98
  version_requirements: !ruby/object:Gem::Requirement
99
99
  requirements:
100
100
  - - "~>"
101
101
  - !ruby/object:Gem::Version
102
- version: '0.1'
102
+ version: 0.3.24
103
103
  - !ruby/object:Gem::Dependency
104
104
  name: mcfly
105
105
  requirement: !ruby/object:Gem::Requirement
@@ -496,7 +496,7 @@ files:
496
496
  - lib/marty/json_schema.rb
497
497
  - lib/marty/lazy_column_loader.rb
498
498
  - lib/marty/logger.rb
499
- - lib/marty/mcfly_query.rb
499
+ - lib/marty/mcfly_model.rb
500
500
  - lib/marty/migrations.rb
501
501
  - lib/marty/monkey.rb
502
502
  - lib/marty/permissions.rb
@@ -1616,6 +1616,7 @@ files:
1616
1616
  - spec/job_helper.rb
1617
1617
  - spec/lib/data_exporter_spec.rb
1618
1618
  - spec/lib/data_importer_spec.rb
1619
+ - spec/lib/delorean_query_spec.rb
1619
1620
  - spec/lib/json_schema_spec.rb
1620
1621
  - spec/lib/logger_spec.rb
1621
1622
  - spec/lib/migrations/vw_marty_postings.sql.expected
@@ -1,189 +0,0 @@
1
- require 'mcfly'
2
-
3
- module Mcfly
4
- module Model
5
-
6
- def self.included(base)
7
- base.send :extend, ClassMethods
8
- end
9
-
10
- module ClassMethods
11
- def clear_lookup_cache!
12
- @LOOKUP_CACHE.clear if @LOOKUP_CACHE
13
- end
14
-
15
- # Implements a VERY HACKY class-based caching mechanism for
16
- # database lookup results. Issues include: cached values are
17
- # ActiveRecord objects. Not sure if these should be shared
18
- # across connections. Query results can potentially be very
19
- # large lists which we simply count as one item in the cache.
20
- # Caching mechanism will result in large processes. Caches are
21
- # not sharable across different Ruby processes.
22
- def cached_delorean_fn(name, options = {}, &block)
23
- @LOOKUP_CACHE ||= {}
24
-
25
- delorean_fn(name, options) do |ts, *args|
26
- cache_key = [name, ts] + args.map{ |a|
27
- a.is_a?(ActiveRecord::Base) ? a.id : a
28
- } unless Mcfly.is_infinity(ts)
29
-
30
- next @LOOKUP_CACHE[cache_key] if
31
- cache_key && @LOOKUP_CACHE.has_key?(cache_key)
32
-
33
- res = block.call(ts, *args)
34
-
35
- if cache_key
36
- # Cache has >1000 items, clear out the oldest 200. FIXME:
37
- # hard-coded, should be configurable. Cache
38
- # size/invalidation should be per lookup and not class.
39
- # We're invalidating cache items simply based on age and
40
- # not usage. This is faster but not as fair.
41
- if @LOOKUP_CACHE.count > 1000
42
- @LOOKUP_CACHE.keys[0..200].each{|k| @LOOKUP_CACHE.delete(k)}
43
- end
44
- @LOOKUP_CACHE[cache_key] = res
45
-
46
- # Since we're caching this object and don't want anyone
47
- # changing it. FIXME: ideally should freeze this object
48
- # recursively.
49
- res.freeze unless res.is_a?(ActiveRecord::Relation)
50
- end
51
- res
52
- end
53
- end
54
-
55
- # FIXME: duplicate code from Mcfly's mcfly_lookup.
56
- def cached_mcfly_lookup(name, options = {}, &block)
57
- cached_delorean_fn(name, options) do |ts, *args|
58
- raise "time cannot be nil" if ts.nil?
59
-
60
- ts = Mcfly.normalize_infinity(ts)
61
-
62
- where("obsoleted_dt >= ? AND created_dt < ?", ts, ts).scoping do
63
- block.call(ts, *args)
64
- end
65
- end
66
- end
67
-
68
- # FIXME: for validation purposes, this mechanism should make
69
- # sure that the allable attrs are not required.
70
- def gen_mcfly_lookup(name, attrs, options={})
71
- raise "bad options" unless options.is_a?(Hash)
72
-
73
- # FIXME: mode should be sent later, not as a part of
74
- # gen_mcfly_lookup. i.e. we just generate the search and the
75
- # mode is applied at runtime by delorean code. That would
76
- # allow lookups to be used in either mode dynamically.
77
- mode = options.fetch(:mode, :first)
78
-
79
- assoc = Set.new(self.reflect_on_all_associations.map(&:name))
80
- attr_names = attrs.keys
81
-
82
- allables = attrs.select {|k, v| v}
83
-
84
- order = allables.keys.reverse.map { |k|
85
- k = "#{k}_id" if assoc.member?(k)
86
- "#{k} NULLS LAST"
87
- }.join(", ")
88
-
89
- qstr = attrs.map {|k, v|
90
- k = "#{k}_id" if assoc.member?(k)
91
- v ? "(#{k} = ? OR #{k} IS NULL)" : "(#{k} = ?)"
92
- }.join(" AND ")
93
-
94
- cached_mcfly_lookup(name, sig: attrs.length+1) do
95
- |t, *attr_list|
96
-
97
- attr_list_ids = attr_list.each_with_index.map {|x, i|
98
- assoc.member?(attr_names[i]) ?
99
- (attr_list[i] && attr_list[i].id) : attr_list[i]
100
- }
101
-
102
- q = self.where(qstr, *attr_list_ids)
103
- q = q.order(order) unless order.empty?
104
- mode = :to_a if mode == :all
105
- mode ? q.send(mode) : q
106
- end
107
- end
108
-
109
- ######################################################################
110
-
111
- # Generates categorization lookups. For instance,
112
- # suppose we have the following in class GFee:
113
- #
114
- # gen_mcfly_lookup_cat :lookup_q,
115
- # [:security_instrument,
116
- # 'Gemini::SecurityInstrumentCategorization',
117
- # :g_fee_category],
118
- # {
119
- # entity: true,
120
- # security_instrument: true,
121
- # coupon: true,
122
- # },
123
- # nil
124
-
125
- # In the above case,
126
- # rel_attr = :security_instrument
127
- # cat_assoc_klass = Gemini::SecurityInstrumentCategorization
128
- # cat_attr = :g_fee_category
129
- # name = :lookup_q
130
- # pc_name = :pc_lookup_q
131
- # pc_attrs = {entity: true, security_instrument: true,
132
- # g_fee_category: true, coupon: true}
133
-
134
- def gen_mcfly_lookup_cat(name, catrel, attrs, options={})
135
- rel_attr, cat_assoc_name, cat_attr = catrel
136
-
137
- raise "#{rel_attr} should be mapped in attrs" if
138
- attrs[rel_attr].nil?
139
-
140
- cat_assoc_klass = cat_assoc_name.constantize
141
-
142
- raise "need lookup method on #{cat_assoc_klass}" unless
143
- cat_assoc_klass.respond_to? :lookup
144
-
145
- # replace rel_attr with cat_attr in attrs
146
- pc_attrs = attrs.each_with_object({}) {|(k, v), h|
147
- h[k == rel_attr ? cat_attr : k] = v
148
- }
149
-
150
- pc_name = "pc_#{name}".to_sym
151
- gen_mcfly_lookup(pc_name, pc_attrs, options)
152
-
153
- lpi = attrs.keys.index rel_attr
154
-
155
- raise "should not include #{cat_attr}" if
156
- attrs.member?(cat_attr)
157
-
158
- raise "need #{rel_attr} argument" unless lpi
159
-
160
- delorean_fn(name, sig: attrs.length+1) do |ts, *args|
161
- # Example: rel is a Gemini::SecurityInstrument instance.
162
- rel = args[lpi]
163
- raise "#{rel_attr} can't be nil" unless rel
164
-
165
- # Assumes there's a mcfly :lookup function on
166
- # cat_assoc_klass.
167
- categorizing_obj = cat_assoc_klass.lookup(ts, rel)
168
- raise "no categorization #{cat_assoc_klass} for #{rel}" unless
169
- categorizing_obj
170
-
171
- pc = categorizing_obj.send(cat_attr)
172
- raise ("#{categorizing_obj} must have assoc." +
173
- " #{cat_attr}/#{rel.inspect}") unless pc
174
-
175
- args[lpi] = pc
176
- self.send(pc_name, ts, *args)
177
- end
178
- end
179
-
180
- end
181
- end
182
- end
183
-
184
- module Mcfly::Controller
185
- # define mcfly user to be Flowscape's current_user.
186
- def user_for_mcfly
187
- find_current_user rescue nil
188
- end
189
- end