marty 0.5.36 → 0.5.38

data/app/models/marty/grid_index_boolean.rb

@@ -0,0 +1,4 @@
+class Marty::GridIndexBoolean < Marty::Base
+  validates_presence_of :created_dt, :data_grid_id, :attr, :index
+  validates_inclusion_of :key, in: [true, false]
+end

data/app/models/marty/grid_index_int4range.rb

@@ -0,0 +1,3 @@
+class Marty::GridIndexInt4range < Marty::Base
+  validates_presence_of :created_dt, :data_grid_id, :attr, :key, :index
+end

data/app/models/marty/grid_index_integer.rb

@@ -0,0 +1,3 @@
+class Marty::GridIndexInteger < Marty::Base
+  validates_presence_of :created_dt, :data_grid_id, :attr, :key, :index
+end

data/app/models/marty/grid_index_numrange.rb

@@ -0,0 +1,3 @@
+class Marty::GridIndexNumrange < Marty::Base
+  validates_presence_of :created_dt, :data_grid_id, :attr, :key, :index
+end

data/app/models/marty/grid_index_string.rb

@@ -0,0 +1,3 @@
+class Marty::GridIndexString < Marty::Base
+  validates_presence_of :created_dt, :data_grid_id, :attr, :index
+end

data/app/models/marty/name_validator.rb

@@ -0,0 +1,15 @@
+class Marty::NameValidator < ActiveModel::Validator
+  def validate(entry)
+    raise "need field option" unless options[:field]
+    field = options[:field].to_sym
+    value = entry.send(field)
+
+    return if value.nil?
+
+    # disallow leading, trailing, >1 internal spaces, special chars (|)
+    if value =~ /\A\s|\s\z|\A.*\s\s.*\z|.*\|.*/
+      entry.errors[field] =
+        I18n.t("activerecord.errors.messages.extraneous_spaces")
+    end
+  end
+end

data/db/migrate/100_create_marty_data_grids.rb

@@ -0,0 +1,16 @@
+class CreateMartyDataGrids < McflyMigration
+  include Marty::Migrations
+
+  def change
+    table_name = "marty_data_grids"
+
+    create_table table_name do |t|
+      t.string :name, null: false
+      # 2-dimensional array which holds grid data
+      t.jsonb :data
+      t.jsonb :metadata
+      t.string :data_type, null: true
+      t.boolean :lenient, null: false, default: false
+    end
+  end
+end

data/db/migrate/101_create_marty_grid_index_numranges.rb

@@ -0,0 +1,27 @@
+class CreateMartyGridIndexNumranges < ActiveRecord::Migration
+  include Marty::Migrations
+
+  def change
+    table_name = "marty_grid_index_numranges"
+
+    # drop deprecated version
+    execute("DROP TABLE IF EXISTS #{table_name}")
+
+    create_table table_name do |t|
+      t.datetime :created_dt, null: false
+      t.references :data_grid, null: false
+      t.string :attr, null: false
+      t.numrange :key, null: false
+      t.integer :index, null: false
+    end
+
+    # FIXME: not sure if this index is appropriate for our queries.
+    # May need to break it up.
+    add_index table_name,
+    [:created_dt, :data_grid_id, :attr],
+    name: "index_#{table_name}"
+    add_index table_name, :key, using: "GIST"
+
+    add_fk table_name, :data_grids
+  end
+end

data/db/migrate/102_create_marty_grid_index_int4ranges.rb

@@ -0,0 +1,27 @@
+class CreateMartyGridIndexInt4ranges < ActiveRecord::Migration
+  include Marty::Migrations
+
+  def change
+    table_name = "marty_grid_index_int4ranges"
+
+    # drop deprecated version
+    execute("DROP TABLE IF EXISTS #{table_name}")
+
+    create_table table_name do |t|
+      t.datetime :created_dt, null: false
+      t.references :data_grid, null: false
+      t.string :attr, null: false
+      t.int4range :key, null: false
+      t.integer :index, null: false
+    end
+
+    # FIXME: not sure if this index is appropriate for our queries.
+    # May need to break it up.
+    add_index table_name,
+    [:created_dt, :data_grid_id, :attr],
+    name: "index_#{table_name}"
+    add_index table_name, :key, using: "GIST"
+
+    add_fk table_name, :data_grids
+  end
+end

data/db/migrate/103_create_marty_grid_index_integers.rb

@@ -0,0 +1,27 @@
+class CreateMartyGridIndexIntegers < ActiveRecord::Migration
+  include Marty::Migrations
+
+  def change
+    table_name = "marty_grid_index_integers"
+
+    # drop deprecated version
+    execute("DROP TABLE IF EXISTS #{table_name}")
+
+    create_table table_name do |t|
+      t.datetime :created_dt, null: false
+      t.references :data_grid, null: false
+      t.string :attr, null: false
+      t.integer :key, array: true, null: false
+      t.integer :index, null: false
+    end
+
+    # FIXME: not sure if this index is appropriate for our queries.
+    # May need to break it up.
+    add_index table_name,
+    [:created_dt, :data_grid_id, :attr],
+    name: "index_#{table_name}"
+    add_index table_name, :key, using: "GIN"
+
+    add_fk table_name, :data_grids
+  end
+end

data/db/migrate/104_create_marty_grid_index_strings.rb

@@ -0,0 +1,27 @@
+class CreateMartyGridIndexStrings < ActiveRecord::Migration
+  include Marty::Migrations
+
+  def change
+    table_name = "marty_grid_index_strings"
+
+    # drop deprecated version
+    execute("DROP TABLE IF EXISTS #{table_name}")
+
+    create_table table_name do |t|
+      t.datetime :created_dt, null: false
+      t.references :data_grid, null: false
+      t.string :attr, null: false
+      t.text :key, array: true
+      t.integer :index, null: false
+    end
+
+    # FIXME: not sure if this index is appropriate for our queries.
+    # May need to break it up.
+    add_index table_name,
+    [:created_dt, :data_grid_id, :attr],
+    name: "index_#{table_name}"
+    add_index table_name, :key, using: "GIN"
+
+    add_fk table_name, :data_grids
+  end
+end

data/db/migrate/105_create_marty_grid_index_booleans.rb

@@ -0,0 +1,27 @@
+class CreateMartyGridIndexBooleans < ActiveRecord::Migration
+  include Marty::Migrations
+
+  def change
+    table_name = "marty_grid_index_booleans"
+
+    # drop deprecated version
+    execute("DROP TABLE IF EXISTS #{table_name}")
+
+    create_table table_name do |t|
+      t.datetime :created_dt, null: false
+      t.references :data_grid, null: false
+      t.string :attr, null: false
+      t.boolean :key, null: false
+      t.integer :index, null: false
+    end
+
+    # FIXME: not sure if this index is appropriate for our queries.
+    # May need to break it up.
+    add_index table_name,
+    [:created_dt, :data_grid_id, :attr],
+    name: "index_#{table_name}"
+    add_index table_name, :key
+
+    add_fk table_name, :data_grids
+  end
+end

data/lib/marty/migrations.rb

@@ -73,12 +73,11 @@ module Marty::Migrations
 
   def self.write_view(target_dir, target_view, klass, jsons, excludes, extras)
     colnames = klass.columns_hash.keys
-    user_id_cols = ["user_id", "o_user_id"]
-    excludes += user_id_cols
+    excludes += ["user_id", "o_user_id"]
     joins = ["join marty_users u on main.user_id = u.id",
              "left join marty_users ou on main.o_user_id = ou.id"]
-    columns = ["concat(u.firstname, ' ', u.lastname) AS user_name",
-               "concat(ou.firstname, ' ', ou.lastname) AS obsoleted_user"]
+    columns = ["u.login AS user_name",
+               "ou.login AS obsoleted_user"]
     jointabs = {}
     colnames.each do |c|
       if jsons[c]

data/lib/marty/version.rb

@@ -1,3 +1,3 @@
 module Marty
-  VERSION = "0.5.36"
+  VERSION = "0.5.38"
 end

data/spec/dummy/app/models/gemini/state.rb

@@ -0,0 +1,19 @@
+class Gemini::State < ActiveRecord::Base
+  extend Marty::Enum
+
+  self.table_name_prefix = "gemini_"
+
+  validates_presence_of :name, :full_name
+  validates_uniqueness_of :name, :full_name
+
+  delorean_fn :lookup, sig: 1 do
+    |name|
+    self.find_by_name(name)
+  end
+
+  def to_s
+    name
+  end
+
+  # FIXME: prevent deletion/update
+end

data/spec/dummy/config/application.rb

@@ -69,7 +69,7 @@ module Dummy
 
     config.marty.system_account = 'marty'
     config.marty.local_password = 'marty'
-    config.marty.class_list = []
+    require 'class_list'
     config.marty.roles = [
                           :admin,
                           :dev,

data/spec/dummy/db/migrate/20160100000038_create_gemini_states.rb

@@ -0,0 +1,8 @@
+class CreateGeminiStates < ActiveRecord::Migration
+  def change
+    create_table :gemini_states do |t|
+      t.string :name, null: false
+      t.string :full_name, null: false
+    end
+  end
+end

data/spec/dummy/db/seeds.rb

@@ -8,3 +8,70 @@ Gemini::MortgageType.create(name: "FHA")
 Gemini::MortgageType.create(name: "VA")
 Gemini::MortgageType.create(name: "USDA/Rural Housing")
 Gemini::StreamlineType.seed
+
+######################################################################
+
+STATES ||=
+  [
+   [ "Alabama",                 "AL" ],
+   [ "Alaska",                  "AK" ],
+   [ "Arizona",                 "AZ" ],
+   [ "Arkansas",                "AR" ],
+   [ "California",              "CA" ],
+   [ "Colorado",                "CO" ],
+   [ "Connecticut",             "CT" ],
+   [ "Delaware",                "DE" ],
+   [ "District Of Columbia",    "DC" ],
+   [ "Florida",                 "FL" ],
+   [ "Georgia",                 "GA" ],
+   [ "Hawaii",                  "HI" ],
+   [ "Idaho",                   "ID" ],
+   [ "Illinois",                "IL" ],
+   [ "Indiana",                 "IN" ],
+   [ "Iowa",                    "IA" ],
+   [ "Kansas",                  "KS" ],
+   [ "Kentucky",                "KY" ],
+   [ "Louisiana",               "LA" ],
+   [ "Maine",                   "ME" ],
+   [ "Maryland",                "MD" ],
+   [ "Massachusetts",           "MA" ],
+   [ "Michigan",                "MI" ],
+   [ "Minnesota",               "MN" ],
+   [ "Mississippi",             "MS" ],
+   [ "Missouri",                "MO" ],
+   [ "Montana",                 "MT" ],
+   [ "Nebraska",                "NE" ],
+   [ "Nevada",                  "NV" ],
+   [ "New Hampshire",           "NH" ],
+   [ "New Jersey",              "NJ" ],
+   [ "New Mexico",              "NM" ],
+   [ "New York",                "NY" ],
+   [ "North Carolina",          "NC" ],
+   [ "North Dakota",            "ND" ],
+   [ "Ohio",                    "OH" ],
+   [ "Oklahoma",                "OK" ],
+   [ "Oregon",                  "OR" ],
+   [ "Pennsylvania",            "PA" ],
+   [ "Rhode Island",            "RI" ],
+   [ "South Carolina",          "SC" ],
+   [ "South Dakota",            "SD" ],
+   [ "Tennessee",               "TN" ],
+   [ "Texas",                   "TX" ],
+   [ "Utah",                    "UT" ],
+   [ "Vermont",                 "VT" ],
+   [ "Virginia",                "VA" ],
+   [ "Washington",              "WA" ],
+   [ "West Virginia",           "WV" ],
+   [ "Wisconsin",               "WI" ],
+   [ "Wyoming",                 "WY" ],
+
+   # US Territories (FIXME: incomplete)
+   [ "American Samoa",          "AS" ],
+   [ "Guam",                    "GU" ],
+   [ "Puerto Rico",             "PR" ],
+   [ "Virgin Islands",          "VI" ],
+  ]
+
+STATES.each { |s| Gemini::State.create(full_name: s[0], name: s[1]) }
+
+######################################################################

data/spec/dummy/lib/class_list.rb

@@ -1,3 +1,3 @@
 module ClassList
-  Rails.configuration.marty.class_list = ["Gemini::FannieBup"]
+  Rails.configuration.marty.class_list = ["Gemini::FannieBup", "Marty::DataGrid"]
 end

data/spec/lib/data_exporter_spec.rb

@@ -1,5 +1,4 @@
 require "spec_helper"
-require "class_list"
 
 module Marty
 

data/spec/lib/data_importer_spec.rb

@@ -1,5 +1,4 @@
 require "spec_helper"
-require "class_list"
 
 module Marty
 

data/spec/lib/migrations/vw_marty_postings.sql.expected

@@ -7,8 +7,8 @@ $$ language plpgsql;
 drop view if exists vw_marty_postings;
 create or replace view vw_marty_postings as
 select
-    concat(u.firstname, ' ', u.lastname) AS user_name,
-    concat(ou.firstname, ' ', ou.lastname) AS obsoleted_user,
+    u.login AS user_name,
+    ou.login AS obsoleted_user,
     main.id,
     main.group_id,
     main.created_dt,

data/spec/models/data_grid_spec.rb

@@ -0,0 +1,614 @@
+require 'spec_helper'
+
+module Marty::DataGridSpec
+  describe DataGrid do
+
+G1 =<<EOS
+state\tstring\tv\t\t
+ltv\tnumrange\tv\t\t
+fico\tnumrange\th\t\t
+
+\t\t>=600<700\t>=700<750\t>=750
+CA\t<=80\t1.1\t2.2\t3.3
+TX|HI\t>80<=105\t4.4\t5.5\t6.6
+NM\t<=80\t1.2\t2.3\t3.4
+MA\t>80<=105\t4.5\t5.6\t
+\t<=80\t11\t22\t33
+EOS
+
+G2 =<<EOS
+units\tinteger\tv\t\t
+ltv\tnumrange\tv\t\t
+cltv\tnumrange\th\t\t
+fico\tnumrange\th\t\t
+
+\t\t>=100<110\t>=110<120\t>=120
+\t\t>=600<700\t>=700<750\t>=750
+1|2\t<=80\t1.1\t2.2\t3.3
+1|2\t>80<=105\t4.4\t5.5\t6.6
+3|4\t<=80\t1.2\t2.3\t3.4
+3|4\t>80<=105\t4.5\t5.6\t6.7
+EOS
+
+G3 = File.open(File.expand_path("../srp_data.csv", __FILE__)).read
+
+G4 =<<EOS
+lenient
+hb_indicator\tboolean\tv
+cltv\tnumrange\th
+
+\t<=60\t>60<=70\t>70<=75\t>75<=80\t>80<=85\t>85<=90\t>90<=95\t>95<=97
+true\t-0.750\t-0.750\t-0.750\t-1.500\t-1.500\t-1.500\t\t
+EOS
+
+G5 =<<EOS
+ltv\tnumrange\tv\t\t
+
+<=115\t-0.375
+>115<=135\t-0.750
+EOS
+
+G6 =<<EOS
+ltv\tnumrange\th
+
+<=115\t>115<=135
+-0.375\t-0.750
+EOS
+
+G7 =<<EOS
+string
+hb_indicator\tboolean\tv
+cltv\tnumrange\th
+
+\t<=60\t>60<=70\t>70<=75\t>75<=80\t>80<=85\t>85<=90\t>90<=95\t>95<=97
+true\tThis\tis\ta\ttest\tof\tstring type\t\t
+EOS
+
+G8 =<<EOS
+Marty::DataGrid
+ltv\tnumrange\tv\t\t
+
+<=115\tG1
+>115<=135\tG2
+>135<=140\tG3
+EOS
+
+G9 =<<EOS
+state\tstring\tv
+ltv\tnumrange\tv
+
+CA|TX\t>80\t123
+\t>80\t456
+EOS
+
+Ga =<<EOS
+dg\tMarty::DataGrid\tv\t\t
+
+G1|G2\t7
+G3\t8
+EOS
+
+Gb =<<EOS
+property_state\tGemini::State\tv\t\t
+
+CA|TX\t70
+GA\t80
+MN\t90
+EOS
+
+Gc =<<EOS
+Marty::DataGrid
+property_state\tGemini::State\tv\t\t
+
+CA|TX\tGb
+EOS
+
+Gd =<<EOS
+hb_indicator\tboolean\tv
+
+true\t456
+false\t123
+EOS
+
+    before(:each) do
+      #Mcfly.whodunnit = Marty::User.find_by_login('marty')
+      marty_whodunnit
+    end
+
+    def lookup_grid_helper(pt, gridname, params, follow=false)
+      dg=Marty::DataGrid.lookup(pt, gridname)
+      res=dg.lookup_grid_distinct_entry(pt, params, nil, follow)
+      [res["result"], res["name"]]
+    end
+
+    describe "validations" do
+      it "a basic data grid should load ok" do
+        dg_from_import("G1", G1)
+        dg_from_import("G2", G2)
+        dg_from_import("G3", G3)
+        dg_from_import("G8", G8)
+        dg_from_import("Ga", Ga)
+
+        expect(Marty::DataGrid.lookup('infinity', "G1").name).to eq "G1"
+        expect(Marty::DataGrid.lookup('infinity', "G2").name).to eq "G2"
+        expect(Marty::DataGrid.lookup('infinity', "G3").name).to eq "G3"
+      end
+
+      it "should not allow dup attr names" do
+        g_bad = G1.sub(/fico/, "ltv")
+
+        expect {
+          dg_from_import("G2", g_bad)
+        }.to raise_error(ActiveRecord::RecordInvalid)
+      end
+
+      it "should not allow dup grid names" do
+        dg_from_import("G1", G1)
+
+        expect {
+          dg_from_import("G1", G2)
+        }.to raise_error(ActiveRecord::RecordInvalid)
+      end
+
+      it "should not allow extra attr rows" do
+        g_bad = "x\tnumrange\th\t\t\n" + G1
+
+        expect {
+          dg_from_import("G2", g_bad)
+        }.to raise_error(RuntimeError)
+      end
+
+      it "should not allow dup row/col key combos" do
+        g_bad = G1 + G1.split("\n").last + "\n"
+        expect {
+          dg_from_import("G2", g_bad)
+        }.to raise_error(ActiveRecord::RecordInvalid)
+
+        g_bad = G2 + G2.split("\n").last + "\n"
+        expect {
+          dg_from_import("G2", g_bad)
+        }.to raise_error(ActiveRecord::RecordInvalid)
+      end
+
+      it "NULL keys are only allowed on string fields" do
+        g_bad = G2.sub(/1\|2/, "")
+
+        expect {
+          dg_from_import("G2", g_bad)
+        }.to raise_error(ActiveRecord::RecordInvalid)
+
+        g_bad = G2.sub(/<=80/, "")
+
+        expect {
+          dg_from_import("G2", g_bad)
+        }.to raise_error(ActiveRecord::RecordInvalid)
+      end
+
+      it "Unknown keys for typed grids should raise error" do
+        g_bad = G8.sub(/G3/, "XXXXX")
+
+        expect {
+          dg_from_import("G8", g_bad)
+        }.to raise_error(RuntimeError)
+
+        g_bad = G8.sub(/DataGrid/, "Division")
+
+        expect {
+          dg_from_import("G8", g_bad)
+        }.to raise_error(RuntimeError)
+      end
+
+      it "Unknown keys for grid headers should raise error" do
+        g_bad = Ga.sub(/G3/, "XXXXX")
+
+        expect {
+          dg_from_import("Ga", g_bad)
+        }.to raise_error(RuntimeError)
+
+        g_bad = Ga.sub(/DataGrid/, "Division")
+
+        expect {
+          dg_from_import("Ga", g_bad)
+        }.to raise_error(RuntimeError)
+      end
+    end
+
+    describe "lookup" do
+      before(:each) do
+        ["G1", "G2", "G3", "G4", "G5", "G6", "G7", "G8", "Ga", "Gb",
+         "Gc", "Gd"].each { |g|
+          dg_from_import(g, "Marty::DataGridSpec::#{g}".constantize)
+        }
+      end
+
+      it "should handle boolean lookups" do
+        res = [true, false].map { |hb_indicator|
+          lookup_grid_helper('infinity',
+                             "Gd",
+                             {"hb_indicator" => hb_indicator,
+                             },
+                            )
+        }
+        expect(res).to eq [[456.0, "Gd"], [123.0, "Gd"]]
+      end
+
+      it "should handle basic lookups" do
+        res = lookup_grid_helper('infinity',
+                                 "G3",
+                                 {"amount" => 160300,
+                                  "state" => "HI",
+                                 },
+                                )
+        expect(res).to eq [1.655,"G3"]
+
+        [3,4].each {
+          |units|
+          res = lookup_grid_helper('infinity',
+                                   "G2",
+                                   {"fico" => 720,
+                                    "units" => units,
+                                    "ltv" => 100,
+                                    "cltv" => 110.1,
+                                   },
+                                  )
+          expect(res).to eq [5.6,"G2"]
+        }
+
+        dg = Marty::DataGrid.lookup('infinity', "G1")
+
+        h = {
+          "fico" => 600,
+          "state" => "RI",
+          "ltv" => 10,
+        }
+
+        res = lookup_grid_helper('infinity', "G1", h)
+        expect(res).to eq [11,"G1"]
+
+        dg.update_from_import("G1", G1.sub(/11/, "111"))
+
+        res = lookup_grid_helper('infinity', "G1", h)
+        expect(res).to eq [111,"G1"]
+      end
+
+      it "should result in error when there are multiple cell hits" do
+        expect {
+          lookup_grid_helper('infinity',
+                             "G2",
+                             {"fico" => 720,
+                              "ltv" => 100,
+                              "cltv" => 110.1,
+                             },
+                            )
+        }.to raise_error(RuntimeError)
+      end
+
+      it "should return nil when matching data grid cell is nil" do
+        res = lookup_grid_helper('infinity',
+                                 "G1",
+                                 {"fico" => 800,
+                                  "state" => "MA",
+                                  "ltv" => 81,
+                                 },
+                                )
+        expect(res).to eq [nil,"G1"]
+      end
+
+      it "should handle string wildcards" do
+        res = lookup_grid_helper('infinity',
+                                 "G1",
+                                 {"fico" => 720,
+                                  "state" => "GU",
+                                  "ltv" => 80,
+                                 },
+                                )
+        expect(res).to eq [22,"G1"]
+      end
+
+      it "should handle matches which also have a wildcard match" do
+        dg_from_import("G9", G9)
+
+        res = lookup_grid_helper('infinity',
+                                 "G9",
+                                 {"state" => "CA", "ltv" => 81},
+                                )
+        expect(res).to eq [123,"G9"]
+
+        res = lookup_grid_helper('infinity',
+                                 "G9",
+                                 {"state" => "GU", "ltv" => 81},
+                                )
+        expect(res).to eq [456,"G9"]
+      end
+
+      it "should handle nil attr values to match wildcard" do
+        dg_from_import("G9", G9)
+
+        res = lookup_grid_helper('infinity',
+                               "G9",
+                               {"state" => nil, "ltv" => 81},
+                               )
+        expect(res).to eq [456,"G9"]
+
+        expect {
+          res = lookup_grid_helper('infinity',
+                                     "G9",
+                                     {"state" => "CA", "ltv" => nil},
+                                    )
+        }.to raise_error(RuntimeError)
+      end
+
+      it "should handle boolean keys" do
+        res = lookup_grid_helper('infinity',
+                                 "G4",
+                                 {"hb_indicator" => true,
+                                  "cltv" => 80,
+                                 },
+                                )
+        expect(res).to eq [-1.5,"G4"]
+
+        res = lookup_grid_helper('infinity',
+                                 "G4",
+                                 {"hb_indicator" => false,
+                                  "cltv" => 80,
+                                 },
+                                )
+        expect(res).to eq [nil,"G4"]
+      end
+
+      it "should handle vertical-only grids" do
+        res = lookup_grid_helper('infinity',
+                                 "G5",
+                                 {"ltv" => 80},
+                                )
+        expect(res).to eq [-0.375,"G5"]
+      end
+
+      it "should handle horiz-only grids" do
+        res = lookup_grid_helper('infinity',
+                                 "G6",
+                                 {"ltv" => 80, "conforming" => true},
+                                )
+        expect(res).to eq [-0.375,"G6"]
+      end
+
+      it "should handle string typed data grids" do
+        expect(Marty::DataGrid.lookup('infinity', "G7").data_type).to eq "string"
+
+        res = lookup_grid_helper('infinity',
+                                 "G7",
+                                 {"hb_indicator" => true,
+                                  "cltv" => 80,
+                                 },
+                                )
+        expect(res).to eq ["test","G7"]
+      end
+
+      it "should handle DataGrid typed data grids" do
+        expect(Marty::DataGrid.lookup('infinity', "G8").data_type).to eq "Marty::DataGrid"
+        g1 = Marty::DataGrid.lookup('infinity', "G1")
+
+        res = lookup_grid_helper('infinity',
+                                 "G8",
+                                 {"ltv" => 80,
+                                 },
+                                )
+        expect(res).to eq [g1,"G8"]
+      end
+
+      it "should handle multi DataGrid lookups" do
+        expect(Marty::DataGrid.lookup('infinity', "G8").data_type).to eq "Marty::DataGrid"
+        g1 = Marty::DataGrid.lookup('infinity', "G1")
+
+        h = {
+          "fico" => 600,
+          "state" => "RI",
+          "ltv" => 10,
+        }
+
+        g1_res = lookup_grid_helper('infinity', "G1", h)
+        expect(g1_res).to eq [11,"G1"]
+
+        res = lookup_grid_helper('infinity',
+                                       "G8",
+                                       h,true
+                                      )
+        expect(g1_res).to eq res
+      end
+
+      it "should handle DataGrid typed data grids" do
+        g1 = Marty::DataGrid.lookup('infinity', "G1")
+
+        res = lookup_grid_helper('infinity',
+                                 "Ga",
+                                 {"dg" => g1,
+                                 },
+                                )
+        expect(res).to eq [7,"Ga"]
+
+        # should be able to lookup bu name as well
+        res = lookup_grid_helper('infinity',
+                                 "Ga",
+                                 {"dg" => "G2",
+                                 },
+                                )
+        expect(res).to eq [7,"Ga"]
+      end
+
+      it "should handle DataGrid typed data grids -- non mcfly" do
+        ca = Gemini::State.find_by_name("CA")
+
+        res = lookup_grid_helper('infinity',
+                                 "Gb",
+                                 {"property_state" => ca,
+                                 },
+                                )
+        expect(res).to eq [70,"Gb"]
+
+        # should be able to lookup bu name as well
+        res = lookup_grid_helper('infinity',
+                                 "Gb",
+                                 {"property_state" => "CA",
+                                 },
+                                )
+        expect(res).to eq [70,"Gb"]
+      end
+
+      it "should return grid data and metadata simple" do
+        pt = 'infinity'
+        expected_data = [[1.1, 2.2, 3.3], [4.4, 5.5, 6.6], [1.2, 2.3, 3.4],
+                         [4.5, 5.6, 6.7]]
+        expected_metadata = [{"dir"=>"v",
+                              "attr"=>"units",
+                              "keys"=>[[1, 2], [1, 2], [3, 4], [3, 4]],
+                              "type"=>"integer"},
+                             {"dir"=>"v",
+                              "attr"=>"ltv",
+                              "keys"=>["[,80]", "(80,105]", "[,80]", "(80,105]"],
+                              "type"=>"numrange"},
+                             {"dir"=>"h",
+                              "attr"=>"cltv",
+                              "keys"=>["[100,110)", "[110,120)", "[120,]"],
+                              "type"=>"numrange"},
+                             {"dir"=>"h",
+                              "attr"=>"fico",
+                              "keys"=>["[600,700)", "[700,750)", "[750,]"],
+                              "type"=>"numrange"}]
+
+        dg = Marty::DataGrid.lookup(pt, 'G2')
+        res = dg.lookup_grid_distinct_entry(pt, {}, nil, true, true)
+        expect(res["data"]).to eq (expected_data)
+        expect(res["metadata"]).to eq (expected_metadata)
+      end
+
+      it "should return grid data and metadata multi (following)" do
+        pt = 'infinity'
+        expected_data =  [[1.1, 2.2, 3.3],[4.4, 5.5, 6.6],[1.2, 2.3, 3.4],
+                          [4.5, 5.6, nil],[11.0, 22.0, 33.0]]
+        expected_metadata = [{"dir"=>"v",
+                              "attr"=>"state",
+                              "keys"=>[["CA"], ["HI", "TX"], ["NM"], ["MA"], nil],
+                              "type"=>"string"},
+                             {"dir"=>"v",
+                              "attr"=>"ltv",
+                              "keys"=>["[,80]", "(80,105]", "[,80]", "(80,105]",
+                                       "[,80]"],
+                              "type"=>"numrange"},
+                             {"dir"=>"h",
+                              "attr"=>"fico",
+                              "keys"=>["[600,700)", "[700,750)", "[750,]"],
+                              "type"=>"numrange"}]
+        dg = Marty::DataGrid.lookup(pt, 'G8')
+        res = dg.lookup_grid_distinct_entry(pt, { "ltv" => 10,
+                                                  "state" => "RI" }, nil, true,
+                                            true)
+        expect(res["data"]).to eq (expected_data)
+        expect(res["metadata"]).to eq (expected_metadata)
+      end
+
+      it "should return grid data and metadata multi (not following)" do
+        pt = 'infinity'
+        expected_data = [["G1"], ["G2"], ["G3"]]
+        expected_metadata = [{"dir"=>"v",
+                              "attr"=>"ltv",
+                              "keys"=>["[,115]", "(115,135]", "(135,140]"],
+                              "type"=>"numrange"}]
+        dg = Marty::DataGrid.lookup(pt, 'G8')
+        res = dg.lookup_grid_distinct_entry(pt, { "ltv" => 10,
+                                                  "state" => "RI" }, nil, false,
+                                            true)
+        expect(res["data"]).to eq (expected_data)
+        expect(res["metadata"]).to eq (expected_metadata)
+      end
+    end
+
+    describe "updates" do
+      it "should be possible to modify a grid referenced from a multi-grid" do
+        dgb = dg_from_import("Gb", Gb, '1/1/2014')
+        dgc = dg_from_import("Gc", Gc, '2/2/2014')
+
+        dgb.update_from_import("Gb", Gb.sub(/70/, "333"), '1/1/2015')
+        dgb.update_from_import("Gb", Gb.sub(/70/, "444"), '1/1/2016')
+
+        res = dgc.lookup_grid_distinct_entry('2/2/2014',
+                                             {"property_state" => "CA"})
+
+        expect(res["result"]).to eq(70)
+
+        res = dgc.lookup_grid_distinct_entry('2/2/2015',
+                                             {"property_state" => "CA"})
+
+        expect(res["result"]).to eq(333)
+
+        res = dgc.lookup_grid_distinct_entry('2/2/2016',
+                                             {"property_state" => "CA"})
+
+        expect(res["result"]).to eq(444)
+      end
+
+      it "should not create a new version if no change has been made" do
+        dg = dg_from_import("G4", G1)
+        dg.update_from_import("G4", G1)
+        expect(Marty::DataGrid.unscoped.where(group_id: dg.group_id).count).to eq 1
+      end
+
+      it "should be able to export and import back grids" do
+        [G1, G2, G3, G4, G5, G6, G7, G8, G9, Ga, Gb].each_with_index do
+          |grid, i|
+          dg = dg_from_import("G#{i}", grid)
+          g1 = dg.export
+          dg = dg_from_import("Gx#{i}", g1)
+          g2 = dg.export
+          expect(g1).to eq g2
+        end
+      end
+
+      it "should be able to externally export/import grids" do
+        load_scripts(nil, Date.today)
+
+        dg = dg_from_import("G1", G1)
+
+        p = posting("BASE", DateTime.tomorrow, '?')
+
+        engine = Marty::ScriptSet.new.get_engine("DataReport")
+        res = engine.evaluate("TableReport",
+                              "result",
+                              {
+                                "pt_name"    => p.name,
+                                "class_name" => "Marty::DataGrid",
+                              },
+                             )
+
+        # FIXME: really hacky removing "" (data_grid) -- This is a bug
+        # in TableReport/CSV generation.
+        res.gsub!(/\"\"/, '')
+        sum = do_import_summary(Marty::DataGrid,
+                                res,
+                                'infinity',
+                                nil,
+                                nil,
+                                ",",
+                               )
+
+        expect(sum).to eq({same: 1})
+
+        res11 = res.sub(/G1/, "G11")
+
+        sum = do_import_summary(Marty::DataGrid,
+                                res11,
+                                'infinity',
+                                nil,
+                                nil,
+                                ",",
+                               )
+
+        expect(sum).to eq({create: 1})
+
+        g1 = Marty::DataGrid.lookup('infinity', "G1")
+        g11 = Marty::DataGrid.lookup('infinity', "G11")
+
+        expect(g1.export).to eq g11.export
+      end
+    end
+  end
+end