marty 2.4.1 → 2.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c3db866b4afffd7cef3f93ecdaf8698dc45bf169
-  data.tar.gz: f1ba792e54124bd28c8a50a5080a3ea556dfe4bb
+  metadata.gz: 4069f0be0ac4faa38738a0aa5f1a76fa56fcbed5
+  data.tar.gz: 832fed4b13056de3a94fbcaebb9fa463fd475f0b
 SHA512:
-  metadata.gz: d40dc1b4535d794f22bd51916449cf48738030780b37a7392abf7ee95ed3ee7f890415090c5b189e271a6f26fa519bf60f8075e9912ff0861896deb73531a366
-  data.tar.gz: 59c87e1a2e69ba55ad37a0b43ae5876487a3226ca1f34b060bf3b889b318f8616cfa3d10f511450a806223a31107d84b4fa5a142af089da7a206ebad1c0151a5
+  metadata.gz: 57cd24df5c095785c4a22d8c801db32a1e504e773bf0d0edce681d53edd71c1b22e55925bcd1fe931dc71457d88b720ad1ad04f1010d50d07df0dfe93caf52e1
+  data.tar.gz: bbcb53ecee094328f5271f5f60b90daa4f606729b196389d0e2316cb14cb53318ef57b603626fca446521aea19a9f734b9b61f1014ec5a7c45e8f4b4bb05bcba

data/lib/marty/json_schema.rb

@@ -35,6 +35,69 @@ module Marty
     end
 
     JSON::Validator.register_validator(self.new)
-  end
 
+    def self.get_numbers(schema)
+      numbers = []
+
+      # traverse the schema, if we find a type: number, add to numbers []
+      trav = lambda { |tree, key, path=[]|
+        return tree.each do|k, v|
+          trav.call(v, k, path + [k])
+        end if tree.is_a?(Hash)
+        numbers << path[0..-2] if key == 'type' && tree == 'number'
+      }
+      trav.call(schema, nil, [])
+
+      # convert the array stuff [ie. "items", "properties"] to :array
+      numbers.map do |num|
+        num.delete("properties")
+        num.map{|n| n=="items" ? :array : n}
+      end
+    end
+
+    def self.fix_numbers(json, numbers)
+
+      # follow path to drill into json
+      drill = lambda {|tree, path|
+        return unless tree
+        key = path.first
+        val = val = tree.send(:[], key) unless key == :array
+        if key == :array
+          # if we are at an array of numbers, fix them
+          if path.length == 1
+            tree.each_with_index do |v, i|
+              tree[i] = v.to_f if v.is_a?(Numeric)
+            end
+          else
+            # this is an array of object so continue to drill down
+            tree.each {|sub| drill.call(sub, path[1..-1])}
+          end
+        elsif path.length == 1
+          # fix a non array field
+          tree.send(:[]=, key, val.to_f) if val.is_a?(Numeric)
+        else
+          # continue drilling
+          drill.call(val, path[1..-1])
+        end
+      }
+      numbers.each {|number| drill.call(json, number)}
+    end
+
+    def self.get_schema(tag, sname, node, attr)
+      begin
+        Marty::ScriptSet.new(tag).get_engine(sname+'Schemas').
+          evaluate(node, attr, {})
+      rescue => e
+        id = "#{sname}/#{node} attrs=#{attr}"
+
+        # the schema DL might not exist at all, or might not define the attr
+        # being requested
+        sch_not_found = ['No such script', "undefined method `#{attr}__D'",
+                         "node #{node} is undefined"]
+        msg = sch_not_found.detect{|msg| e.message.starts_with?(msg)} ?
+                'Schema not defined' : "Problem with schema: #{e.message}"
+        return "Schema error for #{id}: #{msg}"
+      end
+    end
+  end
 end

data/lib/marty/version.rb

@@ -1,3 +1,3 @@
 module Marty
-  VERSION = "2.4.1"
+  VERSION = "2.4.2"
 end

data/other/marty/api/base.rb

@@ -30,6 +30,9 @@ class Marty::Api::Base
   def self.after_evaluate api_params, result
   end
 
+  @@numbers = {}
+  @@schemas = {}
+
   def self.is_authorized? params
     is_secured = Marty::ApiAuth.where(
       script_name: params[:script],
@@ -47,16 +50,25 @@ class Marty::Api::Base
     # prevent script evaluation from modifying passed in params
     params = params.deep_dup
 
+    schema_key = [params[:tag], params[:script], params[:node], params[:attr]]
+    input_schema = nil
+    begin
+      # get_schema will either return a hash with the schema,
+      # or a string with the error
+      input_schema = @@schemas[schema_key] ||=
+                     Marty::JsonSchema.get_schema(*schema_key)
+    rescue => e
+      return {error: e.message}
+    end
+
     # validate input schema
     if config[:input_validated]
-      begin
-        schema = SchemaValidator::get_schema(params)
-      rescue => e
-        return {error: e.message}
-      end
+
+      # must fail if schema not found or some other error
+      return {"error": input_schema} if input_schema.is_a?(String)
 
       begin
-        res = SchemaValidator::validate_schema(schema, params[:params])
+        res = SchemaValidator::validate_schema(input_schema, params[:params])
       rescue NameError
         return {error: "Unrecognized PgEnum for attribute #{params[:attr]}"}
       rescue => e
@@ -68,6 +80,19 @@ class Marty::Api::Base
         schema_errors
     end
 
+    # if schema was found
+    if input_schema.is_a?(Hash)
+      # fix numbers types
+      numbers = @@numbers[schema_key] ||=
+                Marty::JsonSchema.get_numbers(input_schema)
+
+      # modify params in place
+      Marty::JsonSchema.fix_numbers(params[:params], numbers)
+    elsif !input_schema.include?("Schema not defined")
+      # else if some error besides schema not defined, fail
+      return {error: input_schema}
+    end
+
     # get script engine
     begin
       engine = Marty::ScriptSet.new(params[:tag]).get_engine(params[:script])

data/spec/controllers/rpc_controller_spec.rb

@@ -36,6 +36,21 @@ B: A
     p =? 5
 eof
 
+sample_script3err = <<eof
+A:
+    a = 2
+    p =?
+    c = a * 2
+    pc = p + c
+    lc = [pc, pc]
+
+C: A
+    p =? 3
+
+B: A
+    p =? 5
+eof
+
 sample_script4 = <<eof
 import M3
 A: M3::A
@@ -121,8 +136,37 @@ A:
                                    else 'req3'
 
 eof
-
+sample_script11 = <<eof
+A:
+    f1_string  =?
+    f2_bool    =?
+    f3_integer =?
+    f4_number  =?
+    f5_number  =?
+    f6_array   =?
+    f7_array   =?
+
+    v0 = _
+    v1 = {
+           "input": v0,
+           "calc1":  f4_number / 7,
+           "calc1a": f5_number / 7,
+           "calc2":  f3_integer / 7,
+           "calc3": [ f7 / 7 for f7 in f7_array],
+           "calc4": f6_array[0].f14_number / 7,
+           "calc5": f6_array[0].f13_integer / 7,
+           "calc6": [ f7 / 7 for f7 in f6_array[0].f16_array]
+         }
+eof
 script3_schema = <<eof
+A:
+    pc = { "properties" : {
+                  "p" : { "type" : "integer" },
+                }
+            }
+eof
+
+script3err_schema = <<eof
 A:
     pc = { "properties : {
                   "p" : { "type" : "integer" },
@@ -132,6 +176,7 @@ eof
 
 script4_schema = <<eof
 A:
+    a = {}
     d = { "properties" : {
             "p" : { "type" : "integer" },
                 }
@@ -333,7 +378,57 @@ A:
                      new_check
              ] }
 eof
-
+script11_schema = <<eof
+A:
+    properties = {
+       "f1_string"  : {"type": "string"               },
+       "f2_bool"    : {"type": "boolean"              },
+       "f3_integer" : {"type": "integer"              },
+       "f4_number"  : {"type": "number"               },
+       "f5_number"  : {"type": "number", "minimum": 0 },
+       "f6_array"   : {"type": "array",
+                      "items": {
+                          "type"      : "object",
+                          "properties": {
+                             "f11_string"  : {"type": "string"               },
+                             "f12_bool"    : {"type": "boolean"              },
+                             "f13_integer" : {"type": "integer"              },
+                             "f14_number"  : {"type": "number"               },
+                             "f15_number"  : {"type": "number", "minimum": 0 },
+                             "f16_array": {
+                                  "type"        : "array",
+                                  "uniqueItems" : true,
+                                  "minItems"    : 0,
+                                  "maxItems"    : 1,
+                                  "items": {
+                                      "type"       : "number",
+                                      "minimum"    : 0,
+                                      "multipleOf" : 0.001,
+                                      "not": {
+                                          "type": "integer"
+                                      }}}}}},
+       "f7_array": {
+               "type"        : "array",
+               "uniqueItems" : true,
+               "minItems"    : 0,
+               "maxItems"    : 1,
+               "items": {
+                   "type"       : "number",
+                   "minimum"    : 0,
+                   "multipleOf" : 0.001,
+                   "not": {
+                       "type": "integer"
+                   }}},
+      "f8_object": {
+                "type": "object",
+                "properties": {
+                    "f81_number": {"type": "number"},
+                    "f82_array": {"type" : "array",
+                                  "items": {"type": "number"}}}}}
+
+    v1 = { "properties": properties}
+    v0 = { "properties": properties}
+eof
 describe Marty::RpcController do
   before(:each) {
     @routes = Marty::Engine.routes
@@ -351,6 +446,7 @@ describe Marty::RpcController do
                          "M1" => sample_script,
                          "M2" => sample_script.gsub(/a/, "aa").gsub(/b/, "bb"),
                          "M3" => sample_script3,
+                         "M3err" => sample_script3err,
                          "M4" => sample_script4,
                          "M5" => sample_script5,
                          "M6" => sample_script6,
@@ -358,7 +454,9 @@ describe Marty::RpcController do
                          "M8" => sample_script8,
                          "M9" => sample_script9,
                          "M10" => sample_script10,
+                         "M11" => sample_script11,
                          "M3Schemas" => script3_schema,
+                         "M3errSchemas" => script3err_schema,
                          "M4Schemas" => script4_schema,
                          "M5Schemas" => script5_schema,
                          "M6Schemas" => script6_schema,
@@ -366,6 +464,7 @@ describe Marty::RpcController do
                          "M8Schemas" => script8_schema,
                          "M9Schemas" => script9_schema,
                          "M10Schemas" => script10_schema,
+                         "M11Schemas" => script11_schema,
                        }, Date.today + 1.minute)
 
     @p1 = Marty::Posting.do_create("BASE", Date.today + 2.minute, 'a comment')
@@ -676,12 +775,12 @@ describe Marty::RpcController do
           attrs: attr,
           params: params
         }
-    expect = "Schema error for M4/A attrs=h: Problem with schema"
-    expect(response.body).to include("error,#{expect}")
+    expect = "error,Schema error for M4/A attrs=h: Schema not defined"
+    expect(response.body).to include(expect)
   end
 
   it "returns an error message on invalid schema" do
-    Marty::ApiConfig.create!(script: "M3",
+    Marty::ApiConfig.create!(script: "M3err",
                              node: "A",
                              attr: nil,
                              logged: false,
@@ -690,13 +789,13 @@ describe Marty::RpcController do
     params = {"p" => 5}.to_json
     get 'evaluate', params: {
           format: :csv,
-          script: "M3",
+          script: "M3err",
           node: "A",
           attrs: attr,
           params: params
         }
-    expect = "Schema error for M3/A attrs=pc: Problem with schema: "\
-             "syntax error M3Schemas:2\r\n"
+    expect = "Schema error for M3err/A attrs=pc: Problem with schema: "\
+             "syntax error M3errSchemas:2\r\n"
     expect(response.body).to eq("error,#{expect}")
   end
 
@@ -1132,6 +1231,39 @@ describe Marty::RpcController do
     expect(response.body).to match(/"error":"Permission denied"/)
   end
 
+  it "should convert incoming ints in number fields to float" do
+    api = Marty::ApiAuth.new
+    api.app_name = 'TestApp'
+    api.script_name = 'M11'
+    api.save!
+    p = File.expand_path('../../fixtures/json', __FILE__)
+    f = "%s/%s" % [p, "rpc_controller.json"]
+    begin
+      tests = JSON.parse(File.read(f))
+    rescue => e
+      puts "Error parsing #{f}: #{e.message}"
+      raise
+    end
+    aggregate_failures "input coercing" do
+      tests.each_with_index do |t, idx|
+        get 'evaluate', params: {
+              format: :json,
+              script: "M11",
+              node: "A",
+              attrs: t['attr'],
+              api_key: api.api_key,
+              params: t['request'].to_json
+            }
+        resp = JSON.parse(response.body)
+        comp = struct_compare(resp, t['result'], {"float_int_nomatch"=>true})
+        (puts "TEST=#{idx}\n#{comp}"; binding.pry) if comp &&
+                                                      ENV['PRY'] == 'true'
+
+        expect(comp).to be_nil, "input conversion test #{idx}:\n#{comp}"
+      end
+    end
+  end
+
   context "conditional validation" do
     before(:all) do
       Marty::ApiConfig.create!(script: "M10",
@@ -1237,7 +1369,7 @@ describe Marty::RpcController do
 
     it 'returns engine/tag lookup error if script not found' do
       get :evaluate, params: {format: :json, script: 'M1', attrs: "e", tag: 'invalid'}
-      expect(response.body).to match(/"error":"Can't get engine:/)
+      expect(response.body).to match(/bad tag identifier.*invalid/)
       get :evaluate, params: {format: :json, script: 'Invalid', attrs: "e", tag: t1.name}
       expect(response.body).to match(/"error":"Can't get engine:/)
     end

data/spec/fixtures/json/rpc_controller.json

@@ -0,0 +1,175 @@
+[
+    {"attr": "v1",
+     "request": {
+        "f1_string"        : "hi",
+        "f2_bool"          : false,
+        "f3_integer"       : 24,
+        "f4_number"        : 24,
+        "f5_number"        : 24.0,
+        "f6_array"         : [
+            {"f11_string"  : "bye",
+             "f12_bool"    : true,
+             "f13_integer" : 24,
+             "f14_number"  : 24,
+             "f15_number"  : 24.0,
+             "f16_array"   : [24,24,24.0]},
+            {"f11_string"  : "yes",
+             "f12_bool"    : true,
+             "f13_integer" : 24,
+             "f14_number"  : 24,
+             "f15_number"  : 24.0,
+             "f16_array"   : [24]}],
+         "f7_array"         : [24,24,24.0],
+         "f8_object": {"f81_number": 5}
+    },
+     "result": {
+         "input": {
+             "f1_string" : "hi",
+             "f2_bool"   : false,
+             "f3_integer": 24,
+             "f4_number" : 24.0,
+             "f5_number" : 24.0,
+             "f6_array": [
+                 {
+                     "f11_string": "bye",
+                     "f12_bool": true,
+                     "f13_integer": 24,
+                     "f14_number": 24.0,
+                     "f15_number": 24.0,
+                     "f16_array": [
+                         24.0,
+                         24.0,
+                         24.0
+                     ]
+                 },
+                 {
+                     "f11_string": "yes",
+                     "f12_bool": true,
+                     "f13_integer": 24,
+                     "f14_number": 24.0,
+                     "f15_number": 24.0,
+                     "f16_array": [
+                         24.0
+                     ]
+                 }
+             ],
+             "f7_array": [
+                 24.0,
+                 24.0,
+                 24.0
+             ],
+             "f8_object": {"f81_number": 5.0}
+         },
+         "calc1": 3.428571,
+         "calc1a": 3.428571,
+         "calc2": 3,
+         "calc3": [
+             3.428571,
+             3.428571,
+             3.428571
+         ],
+         "calc4": 3.428571,
+         "calc5": 3,
+         "calc6": [
+             3.428571,
+             3.428571,
+             3.428571
+         ]
+     }},
+    {"attr": "v1",
+       "request": {
+        "f1_string"        : "hi",
+        "f2_bool"          : false,
+        "f3_integer"       : 1,
+        "f4_number"        : 1,
+        "f5_number"        : 1,
+        "f6_array"         : [
+            {"f11_string"  : "bye",
+             "f12_bool"    : true,
+             "f13_integer" : 1,
+             "f14_number"  : 1,
+             "f15_number"  : 1,
+             "f16_array"   : [1]}
+            ],
+           "f7_array"         : [1],
+           "f8_object": {"f82_array": [1]}
+
+       },
+     "result": {
+         "input": {
+             "f1_string": "hi",
+             "f2_bool": false,
+             "f3_integer": 1,
+             "f4_number": 1.0,
+             "f5_number": 1.0,
+             "f6_array": [
+                 {
+                     "f11_string": "bye",
+                     "f12_bool": true,
+                     "f13_integer": 1,
+                     "f14_number": 1.0,
+                     "f15_number": 1.0,
+                     "f16_array": [
+                         1.0
+                     ]
+                 }],
+             "f7_array": [
+                 1.0
+             ],
+           "f8_object": {"f82_array": [1.0]}
+         },
+         "calc1": 0.142857,
+         "calc1a": 0.142857,
+         "calc2": 0,
+         "calc3": [
+             0.142857
+         ],
+         "calc4": 0.142857,
+         "calc5": 0,
+         "calc6": [
+             0.142857
+         ]
+     }},
+     {"attr": "v0",
+      "request": {"f4_number": 123},
+      "result": {"f4_number": 123.0}
+     },
+    {"attr": "v0",
+     "request": {"f7_array": [1,2.0,3]},
+      "result": {"f7_array": [1.0,2.0,3.0]}
+    },
+    {"attr": "v0",
+     "request": {"f6_array": [{"f16_array": [1]}]},
+      "result": {"f6_array": [{"f16_array": [1.0]}]}
+    },
+    {"attr": "v0",
+     "request": {"f6_array": [{"f16_array": []}],
+                 "f7_array": []},
+     "result": {"f6_array": [{"f16_array": []}],
+                "f7_array": []}
+    },
+    {"attr": "v0",
+     "request": {},
+     "result": {}
+    },
+    {"attr": "v0",
+     "request": {"f_bool":true},
+     "result": {"f_bool":true}
+    },
+     {"attr": "v0",
+      "request": {"f4_number": "bad"},
+      "result": {"f4_number": "bad"}
+     },
+    {"attr": "v0",
+     "request": {"f7_array": [1,"bad",3]},
+      "result": {"f7_array": [1.0,"bad",3.0]}
+    },
+     {"attr": "v0",
+      "request": {"f4_number": true},
+      "result": {"f4_number": true}
+     },
+    {"attr": "v0",
+     "request": {"f7_array": [1,true,3]},
+      "result": {"f7_array": [1.0,true,3.0]}
+    }
+]

data/spec/fixtures/misc/struct_compare_tests.txt

@@ -0,0 +1,170 @@
+[
+{ "example_number": "0",
+  "v1" : [1],
+  "v2" : [1],
+  "res" : false
+},
+{ "example_number": "1",
+  "v1": [1],
+  "v2": [2],
+  "res": "path=[0] 1 != 2"
+},
+{ "example_number": "2",
+  "v1" : [1,2],
+  "v2" : [1,3],
+  "res" : "path=[1] 2 != 3"
+},
+{ "example_number": "3",
+  "v1": [1,[1,2]],
+  "v2": [1,[1]],
+  "res" : "path=[1] array size mismatch 2 != 1"
+},
+{ "example_number": "4",
+  "v1": [[1,2], "foo", { "abc" : [1,2] }],
+  "v2": [[1,2], "foo", { "abc" : [1,2] }],
+  "res" : false
+},
+{ "example_number": "5",
+  "v1": [[1,2], "foo", { "abc" : [1,2.0] }],
+  "v2": [[1,2], "foo", { "abc" : [1.000,2] }],
+  "res" : false
+},
+{ "example_number": "6",
+  "v1": [[1,2], "foo", { "abc" : [1,{"a": 2}] }],
+  "v2": [[1,2], "foo", { "abc" : [1,{"a": 2}] }],
+  "res" : false
+},
+{ "example_number": "7",
+  "v1": [[1,2], "foo", { "abc" : [1,{"a": 2}] }],
+  "v2": [[1,2], "foo", { "abc" : [1,{"a": 2, "b":3}] }],
+  "res" : "path=[2][\"abc\"][1] hash extra keys [\"b\"]"
+},
+{ "example_number": "8",
+  "v1": [[1,2], "foo", { "abc" : [1,{"a": 2}] }],
+  "v2": [[1,2], "foo", { "abc" : [1,{"b": 2}] }],
+  "res" : "path=[2][\"abc\"][1] hash extra keys [\"a\"]"
+},
+{ "example_number": "9",
+  "v1": [[[1,2,3], {"foo": "bar", "baz": true, "plugh": 2.434, "hash": { "value": [1,2] }}], 7, "value"],
+  "v2": [[[1,2,3], {"foo": "barz", "baz": true, "plugh": 2.434, "hash": { "value": [1,2] }}], 7, "value"],
+  "res": "path=[0][1][\"foo\"] bar != barz"
+},
+{ "example_number": "10",
+  "v1": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],
+  "v2": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],
+  "res": false
+},
+{ "example_number": "11",
+  "v1": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],
+  "v2": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[["one"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],
+  "res": "path=[0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0] class mismatch Integer String"
+},
+{ "example_number": "12",
+  "v1": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],
+  "v2": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],
+  "res": "path=[0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0][0] class mismatch Array Integer"
+},
+{ "example_number": "13",
+  "v1": [{"a": 1, "b": 2, "c": 3, "d": 4},
+         {"a": 1, "b": 2, "c": 3, "d": 4},
+         {"a": 1, "b": 2, "c": 3, "d": 4},
+         {"a": 1, "b": 2, "c": 3, "d": 4}],
+  "v2": [{"a": 1, "b": 2, "c": 3, "d": 4},
+         {"a": 1, "b": 2, "c": 3, "d": 4},
+         {"a": 1, "b": 2, "c": 3, "d": 4}],
+  "res": "path= array size mismatch 4 != 3"
+},
+{ "example_number": "14",
+  "v1": [{"a": 1, "b": 2, "c": 3, "d": 4},
+         {"a": 1, "b": 2, "c": 3, "d": 4},
+         {"a": 1, "b": 2, "c": 3, "d": 4},
+         {"a": 1, "b": 2, "c": 3, "d": 4}],
+  "v2": [{"a": 1, "b": 2, "c": 3, "d": 4},
+         {"a": 1, "b": 2, "c": 3, "d": 4},
+         {"a": 1, "b": 2,         "d": 4},
+         {"a": 1, "b": 2, "c": 3, "d": 4}],
+  "res": "path=[2] hash extra keys [\"c\"]"
+},
+{ "example_number": "15",
+  "v1": [{"a": [{"b": [{"c": [{"d": [1,2.00,3,4]}]}]}]}],
+  "v2": [{"a": [{"b": [{"c": [{"d": [1.00,2,3.00,4.00]}]}]}]}],
+  "res": false
+},
+{ "example_number": "16",
+  "v1": [{"a": [{"b": [{"c": [{"d": [1,2,3,4]}]}]}]}],
+  "v2": [{"a": [{"b": [{"c": [{"d": [1.00,2.10,3.00,4.00]}]}]}]}],
+  "res": "path=[0][\"a\"][0][\"b\"][0][\"c\"][0][\"d\"][1] 2 != 2.1"
+},
+{ "example_number": "17",
+  "v1": [{"a": [{"b": [{"c": [{"d": [1,2.10,3,4]}]}]}]}],
+  "v2": [{"a": [{"b": [{"c": [{"d": [1.00,2,3.00,4.00]}]}]}]}],
+  "res": "path=[0][\"a\"][0][\"b\"][0][\"c\"][0][\"d\"][1] 2.1 != 2"
+},
+{ "example_number": "18",
+  "v1": {"pi": 3.1415926, "e": 2.718281828459, "phi": 1.618034},
+  "v2": {"pi": 3.1415926, "e": 2.718281811111, "phi": 1.618034},
+  "res": false
+},
+{ "example_number": "19",
+  "v1": [[[1,2,3], {"foo": "bar", "baz": false, "plugh": 2.434, "hash": { "value": [1,2] }}], 7, "value"],
+  "v2": [[[1,2,3], {"foo": "bar", "baz": false, "plugh": 2.434, "hash": { "value": [1,2] }}], 7, "value"],
+  "res": false
+},
+{ "example_number": "20",
+  "v1": [[[1,2,3], {"foo": "bar", "baz": null, "plugh": 2.434, "hash": { "value": [1,2] }}], 7, "value"],
+  "v2": [[[1,2,3], {"foo": "bar", "baz": null, "plugh": 2.434, "hash": { "value": [1,2] }}], 7, "value"],
+  "res": false
+},
+{ "example_number": "21",
+  "v1": [[[1,2,3], {"foo": "bar", "baz": null, "plugh": 2.434, "hash": { "value": [1,2] }}], 7, "value"],
+  "v2": [[[1,2,3], {"foo": "bar", "baz": false, "plugh": 2.434, "hash": { "value": [1,2] }}], 7, "value"],
+  "res": "path=[0][1][\"baz\"] class mismatch NilClass FalseClass"
+},
+{ "example_number": "22",
+  "v1": [[[1,2,3], {"foo": "bar", "baz": true, "plugh": 2.434, "hash": { "value": [1,false] }}], 7, "value"],
+  "v2": [[[1,2,3], {"foo": "bar", "baz": true, "plugh": 2.434, "hash": { "value": [1,null] }}], 7, "value"],
+  "res": "path=[0][1][\"hash\"][\"value\"][1] class mismatch FalseClass NilClass"
+},
+{ "example_number": "23",
+  "v1": {"pi": 3.1415926, "e": 2.7182818, "phi": 1.618034},
+  "v2": {"pi": 3.1415926, "e": 2.7282818, "phi": 1.618034},
+  "res": "path=[\"e\"] 2.7182818 != 2.7282818"
+},
+{ "example_number": "24",
+  "v1": {"error": "Some error message returned"},
+  "v2": {"pi": 3.1415926, "e": 2.7282818, "phi": 1.618034},
+  "res": "path= hash extra keys [\"error\"]"
+},
+{ "example_number": "25",
+  "v1": {"pi": 3.1415926, "e": 2.7282818, "phi": 1.618034},
+  "v2": {"error": "Some error message returned"},
+  "res": "path= hash extra keys [\"pi\", \"e\", \"phi\"]"
+},
+{ "example_number": "26",
+  "v1": {"error": "Another error message returned"},
+  "v2": [{"pi": 3.1415926, "e": 2.7282818, "phi": 1.618034}],
+  "res": "Another error message returned"
+},
+{ "example_number": "27",
+  "v1": [{"pi": 3.1415926, "e": 2.7282818, "phi": 1.618034}],
+  "v2": {"error": "Yet another error message returned"},
+  "res": "Yet another error message returned"
+},
+{ "example_number": "28",
+  "v1": {"three": 3},
+  "v2": {"three": 3.0},
+  "res": false
+},
+{ "example_number": "29",
+  "v1": {"three": 3},
+  "v2": {"three": 3.0},
+  "res": "path=[\"three\"] class mismatch Integer Float",
+  "cmp_opts": {"float_int_nomatch": true}
+},
+{ "example_number": "30",
+  "v1": {"three": 3, "dummy": "hi", "dummy2": 1},
+  "v2": {"three": 3.0, "dummy": "bye", "dummy2": 2},
+  "res": false,
+  "cmp_opts": {"float_int_nomatch": false, "ignore": ["dummy", "dummy2"]}
+}
+]

data/spec/lib/struct_compare_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe "StructCompare" do
+  it "compares correctly" do
+    p = File.expand_path('../../fixtures/misc', __FILE__)
+    fname = "%s/%s" % [p, 'struct_compare_tests.txt']
+    aggregate_failures 'struct_compare' do
+      data = JSON.parse(File.read(fname))
+      data.each do |ex|
+        args = [ex["v1"], ex["v2"], ex['cmp_opts']].compact
+        comp = struct_compare(*args)
+        comparison = comp.nil? ? false : comp
+        num = ex["example_number"]
+        binding.pry if comparison != ex['res']
+        expect(comparison).to eq(ex["res"]), "Test ##{num} failed: #{comparison}"
+      end
+    end
+  end
+end

data/spec/support/structure_compare.rb

@@ -1,5 +1,5 @@
 module Marty::RSpec::StructureCompare
-  def self.struct_compare_all(v1raw, v2raw, path=[], errs=[])
+  def self.struct_compare_all(v1raw, v2raw, key=nil, cmp_opts={}, path=[], errs=[])
     pathstr = path.map(&:to_s).join
     v1,v2 = [v1raw, v2raw].map { |v| v.class == ActiveSupport::TimeWithZone ?
                                    DateTime.parse(v.to_s) : v }
@@ -12,18 +12,22 @@ module Marty::RSpec::StructureCompare
 
     return errs + ["path=#{pathstr} class mismatch #{v1.class} #{v2.class}"] unless
       v1.class == v2.class ||
-      [v1,v2].map(&:class).to_set == Set.new([Integer, Float])
+      (!cmp_opts["float_int_nomatch"] &&
+       [v1,v2].map(&:class).to_set == Set.new([Integer, Float]))
 
+    override = (cmp_opts["ignore"]||[]).include?(key)
     case v1
     when String
       return errs + ["path=#{pathstr} #{v1} != #{v2}"] unless
         v1 == v2 ||
         Regexp.new('\A'+v1+'\z').match(v2) ||
-        Regexp.new('\A'+v2+'\z').match(v1)
+        Regexp.new('\A'+v2+'\z').match(v1) ||
+        override
     when Integer, DateTime, TrueClass, FalseClass, NilClass
-      return errs + ["path=#{pathstr} #{v1} != #{v2}"] if v1 != v2
+      return errs + ["path=#{pathstr} #{v1} != #{v2}"] if v1 != v2 && !override
     when Float
-      return errs + ["path=#{pathstr} #{v1} != #{v2}"] if v1.round(6) != v2.round(6)
+      return errs + ["path=#{pathstr} #{v1} != #{v2}"] if
+        v1.round(6) != v2.round(6) && !override
     when Hash
       v1_v2, v2_v1 = v1.keys-v2.keys, v2.keys-v1.keys
 
@@ -31,13 +35,16 @@ module Marty::RSpec::StructureCompare
       errs.append("path=#{pathstr} hash extra keys #{v2_v1}") unless v2_v1.empty?
 
       return errs + v1.map do |childkey, childval|
-        struct_compare_all(childval, v2[childkey], path + [[childkey]], [])
+        struct_compare_all(childval, v2[childkey], childkey, cmp_opts,
+                           path + [[childkey]], [])
       end.flatten
     when Array
-      errs.append("path=#{pathstr} array size mismatch #{v1.size} != #{v2.size}") if
+      errs.append(
+        "path=#{pathstr} array size mismatch #{v1.size} != #{v2.size}") if
         v1.size != v2.size
       return errs + v1.each_with_index.map do |childval,index|
-        struct_compare_all(childval, v2[index], path + [[index]], [])
+        struct_compare_all(childval, v2[index], nil, cmp_opts, path + [[index]],
+                           [])
       end.flatten
     else
       raise "unhandled #{v1.class}"
@@ -46,16 +53,18 @@ module Marty::RSpec::StructureCompare
   end
 end
 
-def struct_compare(v1raw, v2raw)
+def struct_compare(v1raw, v2raw, cmp_opts={})
   begin
-    res = Marty::RSpec::StructureCompare.struct_compare_all(v1raw, v2raw).first
+    res = Marty::RSpec::StructureCompare.struct_compare_all(v1raw, v2raw, nil,
+                                                            cmp_opts).first
   rescue => e
     e.message
   end
 end
-def struct_compare_all(v1raw, v2raw)
+def struct_compare_all(v1raw, v2raw, cmp_opts={})
   begin
-    Marty::RSpec::StructureCompare.struct_compare_all(v1raw, v2raw)
+    Marty::RSpec::StructureCompare.struct_compare_all(v1raw, v2raw, nil,
+                                                      cmp_opts)
   rescue => e
     e.message
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: marty
 version: !ruby/object:Gem::Version
-  version: 2.4.1
+  version: 2.4.2
 platform: ruby
 authors:
 - Arman Bostani
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-12 00:00:00.000000000 Z
+date: 2018-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -1651,6 +1651,8 @@ files:
 - spec/fixtures/csv/rule/DataGrid.csv
 - spec/fixtures/csv/rule/MyRule.csv
 - spec/fixtures/csv/rule/XyzRule.csv
+- spec/fixtures/json/rpc_controller.json
+- spec/fixtures/misc/struct_compare_tests.txt
 - spec/fixtures/scripts/load_tests/script1.dl
 - spec/fixtures/scripts/load_tests/script2.dl
 - spec/job_helper.rb
@@ -1662,6 +1664,7 @@ files:
 - spec/lib/mcfly_model_spec.rb
 - spec/lib/migrations/vw_marty_postings.sql.expected
 - spec/lib/migrations_spec.rb
+- spec/lib/struct_compare_spec.rb
 - spec/lib/xl_spec.rb
 - spec/lib/xl_styles_spec.rb
 - spec/models/api_auth_spec.rb