marty 13.0.2 → 14.0.0

data/app/services/marty/data_grid/constraint.rb

@@ -19,7 +19,7 @@ module Marty
 
           pt = 'infinity'
           vals = raw_vals.map do |v|
-            DataGrid.parse_fvalue(pt, v, data_type, dt)
+            DataGrid.parse_fvalue(pt, v, data_type, dt, false)
           end
           [[:in?, vals.flatten]]
         end
@@ -56,7 +56,7 @@ module Marty
             err = nil
             begin
               cvt_val = cvt && !data_v.class.in?(rt) ?
-                          [DataGrid.parse_fvalue(pt, data_v, dt, klass)].
+                          [DataGrid.parse_fvalue(pt, data_v, dt, klass, false)].
                             flatten.first : data_v
             rescue StandardError => e
               err = e.message

data/db/migrate/603_add_strict_null_mode_to_data_grids.rb

@@ -0,0 +1,5 @@
+class AddStrictNullModeToDataGrids < ActiveRecord::Migration[5.1]
+  def change
+    add_column :marty_data_grids, :strict_null_mode, :boolean, null: false, default: false
+  end
+end

data/lib/marty/content_handler.rb

@@ -6,6 +6,7 @@ module Marty::ContentHandler
     'html' => ['text/html',                'download'],
     'txt'  => ['text/plain',               'inline'],
     'json' => ['application/json',         'download'],
+    'pdf'  => ['application/pdf',          'download'],
 
     # hacky: default format is JSON
     nil    => ['application/json',         'download'],
@@ -29,7 +30,7 @@ module Marty::ContentHandler
         res = to_zip(data)
       when nil, 'json'
         res, format = data.to_json, 'json'
-      when 'html'
+      when 'html', 'pdf'
         res = data.to_s
       else
         res, format = { error: "Unknown format: #{format}" }.to_json, 'json'
@@ -92,6 +93,7 @@ module Marty::ContentHandler
     res = Zip::OutputStream.write_buffer do |stream|
       to_zip_stream(stream, [], data)
     end
+
     res.string
   end
 end

data/lib/marty/railtie.rb

@@ -6,5 +6,6 @@ module Marty
     config.marty.promise_job_enqueue_hooks = []
     config.marty.redis_url = nil
     config.marty.enable_action_cable = true
+    config.marty.data_grid_plpg_lookups = false
   end
 end

data/lib/marty/util.rb

@@ -1,4 +1,6 @@
 module Marty::Util
+  extend Delorean::Functions
+
   def self.set_posting_id(sid)
     snap = Marty::Posting.find_by(id: sid)
     sid = nil if snap && (snap.created_dt == Float::INFINITY)
@@ -56,6 +58,31 @@ module Marty::Util
     res
   end
 
+  # Returns an array of methods and values that can be applied to a number
+  # in order to check if it's in the given range.
+  # Example: '(1,14]') => [[">", 1.0], ["<=", 14.0]]
+  delorean_fn :pg_range_to_ruby, cache: true do |r|
+    next r if r == 'empty' || r.nil?
+
+    m = pg_range_match(r)
+
+    raise "bad PG range #{r}" unless m
+
+    res = []
+
+    if m[:start] != ''
+      op = m[:open] == '(' ? '>' : '>='
+      res += [[op, m[:start].to_f]]
+    end
+
+    if m[:end] != ''
+      op = m[:close] == ')' ? '<' : '<='
+      res += [[op, m[:end].to_f]]
+    end
+
+    res
+  end
+
   def self.human_to_pg_range(r)
     return r if r == 'empty'
 

data/lib/marty/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Marty
-  VERSION = '13.0.2'
+  VERSION = '14.0.0'
 end

data/spec/models/data_grid_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'benchmark/ips'
 
 module Marty::DataGridSpec # rubocop:disable Metrics/ModuleLength
   describe DataGrid do
@@ -187,11 +188,51 @@ Investor Services\t-0.625
 NOT (Admin Premium Services|Admin Services|Admin Services Plus)\t-1.0
 Admin Services Plus\t-1.625
 Investor Services Acadamy\t-0.5
+EOS
+
+    G1_with_nulls = <<EOS
+strict_null_mode
+state\tstring\tv\t\t
+ltv\tnumrange\tv\t\t
+fico\tnumrange\th\t\t
+
+\t\t>=600<700\t>=700<750\t>=750
+CA\t<=80\t1.1\t2.2\t3.3
+TX|HI\t>80<=105\t4.4\t5.5\t6.6
+NM\t<=80\t1.2\t2.3\t3.4
+MA\t>80<=105\t4.5\t5.6\t
+NULL\t<=80\t11\t22\t33
+EOS
+
+    G1_with_bool_nulls = <<EOS
+strict_null_mode
+bool_state\tboolean\tv\t\t
+ltv\tnumrange\tv\t\t
+fico\tnumrange\th\t\t
+
+\t\t>=600<700\t>=700<750\t>=750
+f\t>80<=105\t4.5\t5.6\t
+NULL\t<=80\t11\t22\t33
+EOS
+
+    G1_with_integer_nulls = <<EOS
+strict_null_mode
+int_state\tinteger\tv\t\t
+ltv\tnumrange\tv\t\t
+fico\tnumrange\th\t\t
+
+\t\t>=600<700\t>=700<750\t>=750
+1\t<=80\t1.1\t2.2\t3.3
+2\t>80<=105\t4.4\t5.5\t6.6
+3\t<=80\t1.2\t2.3\t3.4
+4|5\t>80<=105\t4.5\t5.6\t
+NULL\t<=80\t11\t22\t33
 EOS
 
     before(:each) do
       # Mcfly.whodunnit = Marty::User.find_by_login('marty')
       marty_whodunnit
+      Rails.application.config.marty.data_grid_plpg_lookups = false
     end
 
     def lookup_grid_helper(pt, gridname, params, follow = false, distinct = true)
@@ -213,6 +254,110 @@ EOS
           dg_from_import('Gh', Gh + "\t\t\n")
         end.to raise_error(RuntimeError)
       end
+
+      it 'show not allow import NULL fields unless strict_null_mode is on' do
+        expect do
+          dg_from_import(
+            'G1_with_nulls',
+            G1_with_nulls.gsub("strict_null_mode\n", '')
+          )
+        end.to raise_error(
+          /NULL is not supported in grids without strict_null_mode/
+        )
+      end
+
+      it 'should import wildcards' do
+        dg = dg_from_import('G1', G1)
+        state_attr = dg.metadata.find { |key| key['attr'] == 'state' }
+        expect(state_attr['keys'].last).to be nil
+        expect(state_attr['wildcards'].last).to be true
+        expect(state_attr['wildcards']).to eq [false, false, false, false, true]
+      end
+
+      it 'allows to import NULL values in string fields' do
+        dg = dg_from_import('G1_with_nulls', G1_with_nulls)
+        state_attr = dg.metadata.find { |key| key['attr'] == 'state' }
+        expect(state_attr['keys'].last).to be nil
+        expect(state_attr['wildcards'].last).to be false
+
+        # FIXME: do we actually need mixing nulls with values?
+        dg = dg_from_import(
+          'G1_with_nulls2',
+          G1_with_nulls.sub('NULL', 'NY|NULL')
+        )
+        state_attr = dg.metadata.find { |key| key['attr'] == 'state' }
+        expect(state_attr['keys'].last).to eq [nil, 'NY']
+        expect(state_attr['wildcards'].last).to be false
+
+        dg = dg_from_import(
+          'G1_with_nulls3',
+          G1_with_nulls.sub('NULL', 'NOT (NULL)')
+        )
+
+        state_attr = dg.metadata.find { |key| key['attr'] == 'state' }
+        expect(state_attr['keys'].last).to be nil
+        expect(state_attr['wildcards'].last).to be false
+        expect(state_attr['nots'].last).to be true
+
+        dg = dg_from_import(
+          'G1_with_nulls4',
+          G1_with_nulls.sub('NULL', 'NOT (NY|NULL)')
+        )
+
+        state_attr = dg.metadata.find { |key| key['attr'] == 'state' }
+        expect(state_attr['keys'].last).to eq [nil, 'NY']
+        expect(state_attr['wildcards'].last).to be false
+        expect(state_attr['nots'].last).to be true
+      end
+
+      it 'allows to import NULL values in integer field' do
+        dg = dg_from_import('G1_with_integer_nulls', G1_with_integer_nulls)
+        state_attr = dg.metadata.find { |key| key['attr'] == 'int_state' }
+        expect(state_attr['keys'].last).to be nil
+        expect(state_attr['wildcards'].last).to be false
+
+        dg = dg_from_import(
+          'G1_with_integer_nulls2',
+          G1_with_integer_nulls.sub('NULL', '6|NULL')
+        )
+
+        state_attr = dg.metadata.find { |key| key['attr'] == 'int_state' }
+        expect(state_attr['keys'].last).to eq [nil, 6]
+        expect(state_attr['nots'].last).to be false
+        expect(state_attr['wildcards'].last).to be false
+
+        dg = dg_from_import(
+          'G1_with_integer_nulls3',
+          G1_with_integer_nulls.sub('NULL', 'NOT (NULL)')
+        )
+
+        state_attr = dg.metadata.find { |key| key['attr'] == 'int_state' }
+        expect(state_attr['keys'].last).to be nil
+        expect(state_attr['nots'].last).to be true
+        expect(state_attr['wildcards'].last).to be false
+
+        dg = dg_from_import(
+          'G1_with_integer_nulls4',
+          G1_with_integer_nulls.sub('NULL', 'NOT (6|NULL)')
+        )
+
+        state_attr = dg.metadata.find { |key| key['attr'] == 'int_state' }
+        expect(state_attr['keys'].last).to eq [nil, 6]
+        expect(state_attr['nots'].last).to be true
+        expect(state_attr['wildcards'].last).to be false
+      end
+
+      it 'allows to import NULL values in boolean field' do
+        dg = dg_from_import('G1_with_bool_nulls', G1_with_bool_nulls)
+        state_attr = dg.metadata.find { |key| key['attr'] == 'bool_state' }
+        expect(state_attr['keys'].last).to be nil
+        expect(state_attr['wildcards'].last).to be false
+
+        dg = dg_from_import('G1_with_bool_nulls2', G1_with_bool_nulls.sub('NULL', 'NOT (NULL)'))
+        state_attr = dg.metadata.find { |key| key['attr'] == 'bool_state' }
+        expect(state_attr['keys'].last).to be nil
+        expect(state_attr['nots'].last).to be true
+      end
     end
 
     describe 'validations' do
@@ -306,7 +451,7 @@ EOS
 
       before(:each) do
         %w[G1 G2 G3 G4 G5 G6 G7 G8 Ga Gb
-           Gc Gd Ge Gf Gg Gh Gj Gl].each do |g|
+           Gc Gd Ge Gf Gg Gh Gj Gl G1_with_nulls].each do |g|
           dg_from_import(g, "Marty::DataGridSpec::#{g}".constantize)
         end
       end
@@ -351,6 +496,24 @@ EOS
         end
       end
 
+      it 'should cast types' do
+        res = Marty::DataGrid.lookup_grid_h(pt, 'Gf', { 'i' => 13, 'n' => 15 }, true)
+        expect(res).to eq('N')
+
+        res = Marty::DataGrid.lookup_grid_h(pt, 'Gf', { 'i' => '13', 'n' => '15' }, true)
+        expect(res).to eq('N')
+
+        res = Marty::DataGrid.lookup_grid_h(pt, 'Gf', { 'b' => 'true', 'i4' => '6' }, false)
+        expect(res).to eq('Y')
+
+        res = Marty::DataGrid.lookup_grid_h(pt, 'Gg', { 'i1' => 2, 'i2' => 1 }, false)
+        expect(res).to eq(1)
+
+        dg_from_import('G9', G9)
+        res = Marty::DataGrid.lookup_grid_h(pt, 'G9', { 'state' => 4, 'ltv' => 81 }, false)
+        expect(res).to eq(456)
+      end
+
       it 'should handle ambiguous lookups' do
         h1 = {
           'property_state' => 'NY',
@@ -454,7 +617,7 @@ EOS
                              'ltv' => 100,
                              'cltv' => 110.1,
                             )
-        end.to raise_error(RuntimeError)
+        end.to raise_error(RuntimeError, /matches > 1/)
       end
 
       it 'should return nil when matching data grid cell is nil' do
@@ -474,9 +637,81 @@ EOS
                                  'state' => 'GU',
                                  'ltv' => 80,
                                 )
+
         expect(res).to eq [22, 'G1']
       end
 
+      it 'should treat nil as missing attr' do
+        expect do
+          res = lookup_grid_helper('infinity',
+                                   'G1',
+                                   'fico' => 720,
+                                   'state' => 'NM',
+                                   'ltv' => 80,
+                                  )
+        end.to raise_error(RuntimeError, /matches > 1/)
+
+        expect do
+          res = lookup_grid_helper('infinity',
+                                   'G1',
+                                   'fico' => 720,
+                                   'ltv' => 80,
+                                  )
+        end.to raise_error(RuntimeError, /matches > 1/)
+
+        expect do
+          res = lookup_grid_helper('infinity',
+                                   'G1',
+                                   'fico' => 720,
+                                   'state' => nil,
+                                   'ltv' => 80,
+                                  )
+        end.to raise_error(RuntimeError, /matches > 1/)
+      end
+
+      it 'should handle string NULLS' do
+        res = lookup_grid_helper('infinity',
+                                 'G1_with_nulls',
+                                 'fico' => 720,
+                                 'state' => nil,
+                                 'ltv' => 80,
+                                )
+
+        expect(res).to eq [22, 'G1_with_nulls']
+
+        expect do
+          lookup_grid_helper('infinity',
+                             'G1_with_nulls',
+                             'fico' => 720,
+                             'state' => 'BLABLA',
+                             'ltv' => 80,
+                            )
+        end.to raise_error(/Data Grid lookup failed/)
+
+        dg = dg_from_import(
+          'G1_with_nulls2',
+          G1_with_nulls.sub('NULL', 'NY|NULL')
+        )
+
+        res = lookup_grid_helper('infinity',
+                                 dg.name,
+                                 'fico' => 720,
+                                 'state' => nil,
+                                 'ltv' => 80,
+                                )
+
+        expect(res).to eq [22, dg.name]
+
+        res = lookup_grid_helper('infinity',
+                                 dg.name,
+                                 'fico' => 720,
+                                 'state' => 'NY',
+                                 'ltv' => 80,
+                                )
+
+        expect(res).to eq [22, dg.name]
+      end
+
       it 'should handle matches which also have a wildcard match' do
         dg_from_import('G9', G9)
 
@@ -485,7 +720,7 @@ EOS
                                    'G9',
                                    'state' => 'CA', 'ltv' => 81,
                                   )
-        end.to raise_error(RuntimeError)
+        end.to raise_error(RuntimeError, /matches > 1/)
 
         res = lookup_grid_helper('infinity',
                                  'G9',
@@ -494,30 +729,42 @@ EOS
         expect(res).to eq [456, 'G9']
       end
 
-      it 'should raise on nil attr values' do
+      # it 'should raise on nil attr values' do
+      #   next
+      #   dg_from_import('G9', G9)
+      #
+      #   expect do
+      #     lookup_grid_helper('infinity',
+      #                        'G9',
+      #                        'ltv' => 81,
+      #                       )
+      #   end.to raise_error(/matches > 1/)
+      #
+      #   err = /Data Grid lookup failed/
+      #   expect do
+      #     lookup_grid_helper('infinity',
+      #                        'G9',
+      #                        { 'state' => 'CA', 'ltv' => nil },
+      #                        false, false)
+      #   end.to raise_error(err)
+      #
+      #   res = lookup_grid_helper('infinity',
+      #                            'G9',
+      #                            { 'state' => nil, 'ltv' => 81 },
+      #                            false, false)
+      #
+      #   expect(res).to eq [456, 'G9']
+      # end
+
+      it 'should raise if nothing was found' do
         dg_from_import('G9', G9)
 
         expect do
           lookup_grid_helper('infinity',
                              'G9',
-                             'ltv' => 81,
+                             'ltv' => 80,
                             )
-        end.to raise_error(/matches > 1/)
-
-        err = /Data Grid lookup failed/
-        expect do
-          lookup_grid_helper('infinity',
-                             'G9',
-                             { 'state' => 'CA', 'ltv' => nil },
-                             false, false)
-        end.to raise_error(err)
-
-        res = lookup_grid_helper('infinity',
-                                 'G9',
-                                 { 'state' => nil, 'ltv' => 81 },
-                                 false, false)
-
-        expect(res).to eq [456, 'G9']
+        end.to raise_error(/Data Grid lookup failed/)
       end
 
       it 'should handle boolean keys' do
@@ -635,21 +882,25 @@ EOS
                                'attr' => 'units',
                                'keys' => [[1, 2], [1, 2], [3, 4], [3, 4]],
                                'nots' => [false, false, false, false],
+                               'wildcards' => [false, false, false, false],
                                'type' => 'integer' },
                              { 'dir' => 'v',
                                'attr' => 'ltv',
                                'keys' => ['[,80]', '(80,105]', '[,80]', '(80,105]'],
                                'nots' => [false, false, false, false],
+                               'wildcards' => [false, false, false, false],
                                'type' => 'numrange' },
                              { 'dir' => 'h',
                                'attr' => 'cltv',
                                'keys' => ['[100,110)', '[110,120)', '[120,]'],
                                'nots' => [false, false, false],
+                               'wildcards' => [false, false, false],
                                'type' => 'numrange' },
                              { 'dir' => 'h',
                                'attr' => 'fico',
                                'keys' => ['[600,700)', '[700,750)', '[750,]'],
                                'nots' => [false, false, false],
+                               'wildcards' => [false, false, false],
                                'type' => 'numrange' }]
 
         dgh = Marty::DataGrid.lookup_h(pt, 'G2')
@@ -666,17 +917,20 @@ EOS
                                'attr' => 'state',
                                'keys' => [['CA'], ['HI', 'TX'], ['NM'], ['MA'], nil],
                                'nots' => [false, false, false, false, false],
+                               'wildcards' => [false, false, false, false, true],
                                'type' => 'string' },
                              { 'dir' => 'v',
                                'attr' => 'ltv',
                                'keys' => ['[,80]', '(80,105]', '[,80]', '(80,105]',
                                           '[,80]'],
                                 'nots' => [false, false, false, false, false],
+                                'wildcards' => [false, false, false, false, false],
                                'type' => 'numrange' },
                              { 'dir' => 'h',
                                'attr' => 'fico',
                                'keys' => ['[600,700)', '[700,750)', '[750,]'],
                                'nots' => [false, false, false],
+                               'wildcards' => [false, false, false],
                                'type' => 'numrange' }]
         dgh = Marty::DataGrid.lookup_h(pt, 'G8')
         res = Marty::DataGrid.lookup_grid_distinct_entry_h(pt,
@@ -694,6 +948,7 @@ EOS
                                'attr' => 'ltv',
                                'keys' => ['[,115]', '(115,135]', '(135,140]'],
                                'nots' => [false, false, false],
+                               'wildcards' => [false, false, false],
                                'type' => 'numrange' }]
         dgh = Marty::DataGrid.lookup_h(pt, 'G8')
         res = Marty::DataGrid.lookup_grid_distinct_entry_h(pt,
@@ -945,6 +1200,65 @@ EOS
           false\t\t>10\t\t#{values3[2]}
         EOS
     end
+
+    describe 'performance' do
+      before(:each) do
+        %w[G1 Gf Gl].each do |g|
+          dg_from_import(g, "Marty::DataGridSpec::#{g}".constantize)
+        end
+      end
+
+      after do
+        Rails.application.config.marty.data_grid_plpg_lookups = false
+      end
+
+      let(:pt) { 'infinity' }
+
+      grid_data = {
+        'Gf' => { 'b' => true },
+        'G1' => {
+          'fico' => 600,
+          'state' => 'RI',
+          'ltv' => 10,
+        },
+        'Gl' => {
+          'fha_203k_option2' => 'Not Existing Services'
+        }
+      }
+
+      grid_data.each_with_index do |(grid, params), index|
+        it "ruby lookup is faster than plpgsql #{index}" do
+          bm = Benchmark.ips do |x|
+            x.report('postgres') do
+              Rails.application.config.marty.data_grid_plpg_lookups = true
+              res = Marty::DataGrid.lookup_grid_h(pt, grid, params, false)
+            end
+
+            x.report('ruby') do
+              Rails.application.config.marty.data_grid_plpg_lookups = false
+              res = Marty::DataGrid.lookup_grid_h(pt, grid, params, false)
+            end
+
+            x.compare!
+          end
+
+          h = bm.entries.each_with_object({}) do |e, hh|
+            hh[e.label] = e.stats.central_tendency
+          end
+
+          factor = h['ruby'] / h['postgres']
+
+          if ENV['CI'] == 'true'
+            # Performance drops down in CI, probably due to running postgres
+            # in a separate container utilizing it's own CPU core.
+            expect(factor).to be > 0.8
+          else
+            expect(factor).to be > 1.02
+          end
+        end
+      end
+    end
+
     describe 'constraint' do
       it 'constraint' do
         Mcfly.whodunnit = system_user