marty 1.0.33 → 1.0.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f81ea6f27f1cb8c9671b8e4dc1c6d32ab3e782e
-  data.tar.gz: 839a0aa364eb2a4991d3b49bbb729e63c620ea45
+  metadata.gz: 751328c7a7c786984962edb15f66a8aa2fa680ce
+  data.tar.gz: 08d5426a9b837d2f4da1f759c6e27c0a9f7c1c5a
 SHA512:
-  metadata.gz: a2bde0eae1e81e2189f3d5d53c88a25d6c317f40fe46309a0b330f0fdb71c77414230bd62a211f98f348c20807985b167c8a8bf38e9e8ce8a0b7105630c1060f
-  data.tar.gz: 3b49c4026098a1ff32e15c8684096fd8fe786361dafca179ce60eeb793fc1166fbd91fd7193d4c7b39fc479269bf16ca0d99d93751fac61feb1755323b702e4e
+  metadata.gz: af123dfccdbd3a075eb489979afa78ffd21bbcfd3615ddcc6b5f764be76d61394e1aab0d853f35f17c327effd9298f0c293c946962ea52fdec764e137dd57713
+  data.tar.gz: 40ab7f6dfa2987c2c580a3c5cd4dd0f6baee3b48eb3225572c4ce17dfa6d58df591fe74db745daf421b8e511eff2df0e1fbf47c865daa28114a24c382a2be66d

data/app/components/marty/config_view.rb

@@ -22,7 +22,7 @@ class Marty::ConfigView < Marty::Grid
   end
   def my_jsonb_pretty_getter
     lambda { |r| v = Marty::Config[r.key]
-      v && JSON.pretty_generate(v) || '' }
+      v && (JSON.pretty_generate(v) rescue v.to_json) || '' }
   end
 
   def my_jsonb_setter

data/app/controllers/marty/rpc_controller.rb

@@ -83,10 +83,16 @@ class Marty::RpcController < ActionController::Base
               :errors_as_objects  => true,
               :version            => Marty::JsonSchema::RAW_URI }
       to_append = {"\$schema" => Marty::JsonSchema::RAW_URI}
-      schemas.each do |attr, schema|
-        err = JSON::Validator.fully_validate(schema.merge(to_append), params, opt)
-        validation_error[attr] = err.map{ |e| e[:message] } if err.size > 0
-        err_count += err.size
+      schemas.each do |attr, sch|
+        begin
+          er = JSON::Validator.fully_validate(sch.merge(to_append), params, opt)
+        rescue NameError
+          return {error: "Unrecognized PgEnum for attribute #{attr}"}
+        rescue => ex
+          return {error: ex.message}
+        end
+        validation_error[attr] = er.map{ |e| e[:message] } if er.size > 0
+        err_count += er.size
       end
     end
     return {error: "Error(s) validating: #{validation_error}"} if err_count > 0

data/lib/marty/data_conversion.rb

@@ -46,7 +46,7 @@ class Marty::DataConversion
       end
     when :string, :text, :enum
       v
-    when :enum_array
+    when :enum_array, :string_array, :integer_array
       "'{#{v}}'"
     when :integer
       v.to_i

data/lib/marty/json_schema.rb

@@ -19,31 +19,16 @@ module Marty
     end
   end
 
-  class DateTimeFormatAttribute < JSON::Schema::Attribute
-    def self.validate(curr_schema, data, frag, processor, validator, options={})
-      begin
-        DateTime.parse(data).in_time_zone(Rails.configuration.time_zone)
-      rescue
-        msg = "#{self.class.name} error: Can't parse '#{data}' into a DateTime"
-        validation_error( processor,
-                          msg,
-                          frag,
-                          curr_schema,
-                          self,
-                          options[:record_errors])
-      end
-    end
-  end
-
   class JsonSchema < JSON::Schema::Draft4
     RAW_URI = "http://json-schema.org/marty-draft/schema#"
 
     def initialize
       super
       @attributes["pg_enum"] = PgEnumAttribute
-      @attributes["datetime_format"] = DateTimeFormatAttribute
-      @uri = JSON::Util::URI.parse(RAW_URI)
-      @names = ["marty-draft", RAW_URI]
+      @formats["date-time"]  = JSON::Schema::DateTimeFormat
+      @formats["date"]       = JSON::Schema::DateFormat
+      @uri                   = JSON::Util::URI.parse(RAW_URI)
+      @names                 = ["marty-draft", RAW_URI]
     end
 
     JSON::Validator.register_validator(self.new)

data/lib/marty/version.rb

@@ -1,3 +1,3 @@
 module Marty
-  VERSION = "1.0.33"
+  VERSION = "1.0.34"
 end

data/spec/controllers/rpc_controller_spec.rb

@@ -62,6 +62,24 @@ A:
         else 9
 eof
 
+sample_script6 = <<eof
+A:
+    b =?
+    res = b + 1
+eof
+
+sample_script7 = <<eof
+A:
+    b =?
+    res = b
+eof
+
+sample_script8 = <<eof
+A:
+    b =?
+    res = 123
+eof
+
 script3_schema = <<eof
 A:
     pc = { "properties : {
@@ -97,6 +115,30 @@ A:
             }
 eof
 
+script6_schema = <<eof
+A:
+    res = { "properties" : {
+            "b" : { "type" : "float" },
+                }
+            }
+eof
+
+script7_schema = <<eof
+A:
+    res = { "properties" : {
+            "b" : { "pg_enum" : "NonExistantEnum" },
+                }
+            }
+eof
+
+script8_schema = <<eof
+A:
+    res = { "properties" : {
+            "b" : { "pg_enum" : "Gemini::MiDurationType" },
+                }
+            }
+eof
+
 
 describe Marty::RpcController do
   before(:each) {
@@ -117,9 +159,15 @@ describe Marty::RpcController do
                          "M3" => sample_script3,
                          "M4" => sample_script4,
                          "M5" => sample_script5,
+                         "M6" => sample_script6,
+                         "M7" => sample_script7,
+                         "M8" => sample_script8,
                          "M3Schemas" => script3_schema,
                          "M4Schemas" => script4_schema,
                          "M5Schemas" => script5_schema,
+                         "M6Schemas" => script6_schema,
+                         "M7Schemas" => script7_schema,
+                         "M8Schemas" => script8_schema,
                        }, Date.today + 1.minute)
 
     @p1 = Marty::Posting.do_create("BASE", Date.today + 2.minute, 'a comment')
@@ -370,7 +418,7 @@ describe Marty::RpcController do
     expect(response.body).to eq("a,b\r\n123,456\r\n789,101112\r\n")
   end
 
-  it "returns an error message on missing schema script" do
+  it "returns an error message on missing schema script (csv)" do
     Marty::ApiConfig.create!(script: "M1",
                              node: "A",
                              attr: nil,
@@ -389,6 +437,28 @@ describe Marty::RpcController do
     expect(response.body).to eq("error,#{expect}")
   end
 
+  it "returns an error message on missing schema script (json)" do
+    Marty::ApiConfig.create!(script: "M1",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    attrs = ["b"].to_json
+    params = {"a" => 5}.to_json
+    get 'evaluate', {
+      format: :json,
+      script: "M1",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+    expect = "Schema error for M1/A attrs=b: Schema not defined"
+    res_hsh = JSON.parse(response.body)
+    expect(res_hsh.keys.size).to eq(1)
+    expect(res_hsh.keys[0]).to eq("error")
+    expect(res_hsh.values[0]).to eq(expect)
+  end
+
   it "returns an error message on missing attributes in schema script" do
     Marty::ApiConfig.create!(script: "M4",
                              node: "A",
@@ -491,6 +561,30 @@ describe Marty::RpcController do
     expect(response.body).to eq("9\r\n9\r\n")
   end
 
+  it "catches JSON::Validator exceptions" do
+    Marty::ApiConfig.create!(script: "M6",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    attrs = ["res"].to_json
+    params = {"b" => 5.22}.to_json
+    get 'evaluate', {
+      format: :json,
+      script: "M6",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+    expect = 'The property \'#/properties/b/type\' of type string '\
+             'did not match one or more of the required schemas'
+    res_hsh = JSON.parse(response.body)
+    expect(res_hsh.keys.size).to eq(1)
+    expect(res_hsh.keys[0]).to eq("error")
+    expect(res_hsh.values[0]).to eq(expect)
+  end
+
+
   class FruitsEnum
     VALUES=Set['Apple', 'Banana', 'Orange']
   end
@@ -532,6 +626,47 @@ describe Marty::RpcController do
     expect(response.body).to include(expect)
   end
 
+  it "validates schema with a non-existant enum" do
+    Marty::ApiConfig.create!(script: "M7",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    attrs = ["res"].to_json
+    params = {"b" => "MemberOfANonExistantEnum"}.to_json
+    get 'evaluate', {
+      format: :json,
+      script: "M7",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+    expect = "Unrecognized PgEnum for attribute res"
+    res_hsh = JSON.parse(response.body)
+    expect(res_hsh.keys.size).to eq(1)
+    expect(res_hsh.keys[0]).to eq("error")
+    expect(res_hsh.values[0]).to include(expect)
+  end
+
+  it "validates pgenum with capitalization issues" do
+    Marty::ApiConfig.create!(script: "M8",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    skip "pending until a solution is found that handles "\
+         "autoload issues involving constantize"
+    attrs = ["res"].to_json
+    params = {"b" => "Annual"}.to_json
+    get 'evaluate', {
+      format: :json,
+      script: "M8",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+  end
+
   it "should log good req" do
     Marty::ApiConfig.create!(script: "M3",
                              node: "A",

data/spec/dummy/app/models/gemini/mi_duration_type.rb

@@ -0,0 +1,12 @@
+class Gemini::MIDurationType < ActiveRecord::Base
+  extend Marty::PgEnum
+
+  VALUES = Set["Annual",
+               "NotApplicable",
+               "Other",
+               "PeriodicMonthly",
+               "SingleLifeOfLoan",
+               "SingleSpecific",
+               "SplitPremium"]
+
+end

data/spec/dummy/db/migrate/20170725160000_add_misc_arrays_to_lp.rb

@@ -0,0 +1,8 @@
+class AddMiscArraysToLp < ActiveRecord::Migration
+  def change
+    add_column :gemini_loan_programs, :test_int_array, :integer,
+               array: true, null: true
+    add_column :gemini_loan_programs, :test_string_array, :string,
+               array: true, null: true
+  end
+end

data/spec/lib/data_importer_spec.rb

@@ -53,16 +53,17 @@ Conv Fixed 30	2.250	1.123	2.345	12	2012
 EOF
 
 loan_programs =<<EOF
-name	amortization_type	mortgage_type	streamline_type	high_balance_indicator	state_array
-Conv Fixed 30 Year	Fixed	Conventional	Not Streamlined	false	
-Conv Fixed 30 Year HB	Fixed	Conventional	Not Streamlined	true	TN
-Conv Fixed 30 Year DURP <=80	Fixed	Conventional	DURP	false	TN,CT
-Conv Fixed 30 Year DURP <=80 HB	Fixed	Conventional	DURP	true	"CA,NY"
+name	amortization_type	mortgage_type	streamline_type	high_balance_indicator	state_array	test_int_array	test_string_array
+Conv Fixed 30 Year	Fixed	Conventional	Not Streamlined	false		1	foo
+Conv Fixed 30 Year HB	Fixed	Conventional	Not Streamlined	true	TN	"1,2"	foo,bar
+Conv Fixed 30 Year DURP <=80	Fixed	Conventional	DURP	false	TN,CT	1,2,3	"foo,bar"
+Conv Fixed 30 Year DURP <=80 HB	Fixed	Conventional	DURP	true	"CA,NY"		foo,hi mom
 EOF
 
 loan_programs_comma =<<EOF
-name,amortization_type,mortgage_type,state_array,streamline_type,high_balance_indicator
-FHA Fixed 15 Year,Fixed,FHA,"FL,NV,ME",Not Streamlined,false
+name,amortization_type,mortgage_type,state_array,test_string_array,streamline_type,high_balance_indicator
+FHA Fixed 15 Year,Fixed,FHA,"FL,NV,ME","ABC,DEF",Not Streamlined,false
+FHA Fixed 100 Year,Fixed,FHA,"FL,NV,ME","XYZ,""hi mom""",Not Streamlined,false
 EOF
 
 fannie_bup4 =<<EOF
@@ -347,16 +348,24 @@ EOF
       end
     end
 
-    it "should load enum array types" do
+    it "should load array types" do
       Marty::DataImporter.do_import(Gemini::LoanProgram, loan_programs)
       Marty::DataImporter.do_import(Gemini::LoanProgram, loan_programs_comma,
                                     'infinity', nil, nil, ',')
-      lpset = Gemini::LoanProgram.all.pluck(:name, :state_array).to_set
-      expect(lpset).to eq([["Conv Fixed 30 Year", nil],
-                           ["Conv Fixed 30 Year HB", ["TN"]],
-                           ["Conv Fixed 30 Year DURP <=80", ["TN", "CT"]],
-                           ["Conv Fixed 30 Year DURP <=80 HB", ["CA","NY"]],
-                           ["FHA Fixed 15 Year", ["FL","NV","ME"]]
+      lpset = Gemini::LoanProgram.all.pluck(:name, :state_array,
+                                            :test_int_array,
+                                            :test_string_array).to_set
+      expect(lpset).to eq([["Conv Fixed 30 Year", nil, [1], ['foo']],
+                           ["Conv Fixed 30 Year HB", ["TN"], [1, 2],
+                            ['foo', 'bar']],
+                           ["Conv Fixed 30 Year DURP <=80", ["TN", "CT"],
+                            [1, 2, 3], ['foo', 'bar']],
+                           ["Conv Fixed 30 Year DURP <=80 HB", ["CA", "NY"],
+                            nil, ['foo', 'hi mom']],
+                           ["FHA Fixed 15 Year", ["FL","NV","ME"], nil,
+                            ['ABC', 'DEF']],
+                           ["FHA Fixed 100 Year", ["FL","NV","ME"], nil,
+                            ['XYZ', 'hi mom']]
                           ].to_set)
 
     end

data/spec/lib/json_schema_spec.rb

@@ -98,64 +98,135 @@ module Marty
       expect(JSON::Validator.validate(pg_schema_req, data)).to be false
     end
 
-    dt_schema_opt = {
+    ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###
+    ###                      Date Format                        ###
+    ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###
+
+    date_schema_opt = {
       "$schema" => "http://json-schema.org/marty-draft/schema#",
       "properties" => {
         "a" => {
-          "datetime_format" => ""
+          "type"   => "string",
+          "format" => "date"
         }
       }
     }
 
+    it "returns true on a properly formatted date" do
+      data = {"a" => '2017-05-22'}
+      expect(JSON::Validator.validate(date_schema_opt, data)).to be true
+    end
+
+    it "vacuously returns true on a field not validated" do
+      data = {"b" => 'Today is May 22nd'}
+      expect(JSON::Validator.validate(date_schema_opt, data)).to be true
+    end
+
+    it "returns false on an improperly formatted date" do
+      data = {"a" => '2017-05-32'}
+      expect(JSON::Validator.validate(date_schema_opt, data)).to be false
+    end
+
+    it "returns false on an properly formatted datetime" do
+      data = {"a" => '2017-05-22T14:51:44Z'}
+      expect(JSON::Validator.validate(date_schema_opt, data)).to be false
+    end
+
+    it "returns true when an optional date is not supplied" do
+      data = {}
+      expect(JSON::Validator.validate(date_schema_opt, data)).to be true
+    end
+
+    it "returns false when a nil date is passed even when date is optional" do
+      data = {"a" => nil}
+      expect(JSON::Validator.validate(date_schema_opt, data)).to be false
+    end
+
+    date_schema_req = {
+      "$schema" => "http://json-schema.org/marty-draft/schema#",
+      "required" => ["a"],
+      "properties" => {
+        "a" => {
+          "type"   => "string",
+          "format" => "date"
+        }
+      }
+    }
+
+    it "returns false when a required date field is not supplied" do
+      data = {}
+      expect(JSON::Validator.validate(date_schema_req, data)).to be false
+    end
+
+    it "returns false when a nil date is passed when date is required" do
+      data = {"a" => nil}
+      expect(JSON::Validator.validate(date_schema_req, data)).to be false
+    end
+
     ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###
     ###                      DateTime Format                    ###
     ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###
 
+    datetime_schema_opt = {
+      "$schema" => "http://json-schema.org/marty-draft/schema#",
+      "properties" => {
+        "a" => {
+          "type"   => "string",
+          "format" => "date-time"
+        }
+      }
+    }
+
     it "returns true on a properly formatted datetime" do
       data = {"a" => '2017-05-22T14:51:44Z'}
-      expect(JSON::Validator.validate(dt_schema_opt, data)).to be true
+      expect(JSON::Validator.validate(datetime_schema_opt, data)).to be true
     end
 
     it "vacuously returns true on a field not validated" do
       data = {"b" => 'Today is May 22nd'}
-      expect(JSON::Validator.validate(dt_schema_opt, data)).to be true
+      expect(JSON::Validator.validate(datetime_schema_opt, data)).to be true
     end
 
     it "returns false on an improperly formatted datetime" do
       data = {"a" => '2017-30-22T14:51:44Z'}
-      expect(JSON::Validator.validate(dt_schema_opt, data)).to be false
+      expect(JSON::Validator.validate(datetime_schema_opt, data)).to be false
     end
 
     it "returns true when an opt field is not supplied" do
       data = {}
-      expect(JSON::Validator.validate(dt_schema_opt, data)).to be true
+      expect(JSON::Validator.validate(datetime_schema_opt, data)).to be true
     end
 
     it "returns false when a nil dt is passed even when dt is opt" do
       data = {"a" => nil}
-      expect(JSON::Validator.validate(dt_schema_opt, data)).to be false
+      expect(JSON::Validator.validate(datetime_schema_opt, data)).to be false
     end
 
-    dt_schema_req = {
+    datetime_schema_req = {
       "$schema" => "http://json-schema.org/marty-draft/schema#",
       "required" => ["a"],
       "properties" => {
         "a" => {
-          "datetime_format" => ""
+          "type"   => "string",
+          "format" => "date-time"
         }
       }
     }
 
     it "returns false when a required field is not supplied" do
       data = {}
-      expect(JSON::Validator.validate(dt_schema_req, data)).to be false
+      expect(JSON::Validator.validate(datetime_schema_req, data)).to be false
     end
 
     it "returns false when a nil dt is passed when dt is required" do
       data = {"a" => nil}
-      expect(JSON::Validator.validate(dt_schema_req, data)).to be false
+      expect(JSON::Validator.validate(datetime_schema_req, data)).to be false
     end
 
+    ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###
+    ###                 PgEnum & DateTime Format                ###
+    ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###
+
     pg_dt_schema = {
       "$schema" => "http://json-schema.org/marty-draft/schema#",
       "properties" => {
@@ -163,15 +234,12 @@ module Marty
           "pg_enum" => "MammalEnum"
         },
         "b" => {
-          "datetime_format" => ""
+          "type"   => "string",
+          "format" => "date-time"
         },
       }
     }
 
-    ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###
-    ###                 PgEnum & DateTime Format                ###
-    ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###
-
     it "validates both pg_enum and dt format when both are correct" do
       data = {"a" => 'Dog', "b" => '2017-05-22T14:51:44Z'}
       expect(JSON::Validator.validate(pg_dt_schema, data)).to be true
@@ -199,7 +267,8 @@ module Marty
           "pg_enum" => "MammalEnum"
         },
         "b" => {
-          "datetime_format" => ""
+          "type"   => "string",
+          "format" => "date-time"
         },
         "c" => {
           "type" => "integer"
@@ -225,7 +294,8 @@ module Marty
           "pg_enum" => "MammalEnum"
         },
         "b" => {
-          "datetime_format" => ""
+          "type"   => "string",
+          "format" => "date-time"
         },
         "c" => {
           "type" => "integer"
@@ -246,7 +316,8 @@ module Marty
           "pg_enum" => "MammalEnum"
         },
         "b" => {
-          "datetime_format" => ""
+          "type"   => "string",
+          "format" => "date-time"
         },
         "c" => {
           "pg_enum" => "ElectronicsEnum"
@@ -285,7 +356,8 @@ module Marty
           "pg_enum" => "MammalEnum"
         },
         "b" => {
-          "datetime_format" => ""
+          "type"   => "string",
+          "format" => "date-time"
         },
         "c" => {
           "pg_enum" => "ElectronicsEnum"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: marty
 version: !ruby/object:Gem::Version
-  version: 1.0.33
+  version: 1.0.34
 platform: ruby
 authors:
 - Arman Bostani
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-13 00:00:00.000000000 Z
+date: 2017-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -537,6 +537,7 @@ files:
 - spec/dummy/app/models/gemini/head_version.rb
 - spec/dummy/app/models/gemini/helper.rb
 - spec/dummy/app/models/gemini/loan_program.rb
+- spec/dummy/app/models/gemini/mi_duration_type.rb
 - spec/dummy/app/models/gemini/mortgage_type.rb
 - spec/dummy/app/models/gemini/simple.rb
 - spec/dummy/app/models/gemini/state.rb
@@ -571,6 +572,7 @@ files:
 - spec/dummy/db/migrate/20160100000038_create_gemini_states.rb
 - spec/dummy/db/migrate/20160923183516_add_bulk_pricing_event_ops.rb
 - spec/dummy/db/migrate/20170706081300_add_state_array_to_lp.rb
+- spec/dummy/db/migrate/20170725160000_add_misc_arrays_to_lp.rb
 - spec/dummy/db/seeds.rb
 - spec/dummy/delorean/blame_report.dl
 - spec/dummy/delorean/data_report.dl