marty 1.0.35 → 1.0.36

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3adb3fc7a00a1b5f8821771a2105ae9e2788d2af
-  data.tar.gz: 7e9faa47b3d14a1df9dab85c206031ec4dd0452f
+  metadata.gz: 3ee1b4758299664b9d0140ae70dcd1b2c0dddc5e
+  data.tar.gz: 865b7a125ce0321f3ca674936ab74023daa6c6a9
 SHA512:
-  metadata.gz: 7ac9b7eb9d94c4fd78ab1869b6ebf8bde560a4d66f59a91e0c98c53a901b45ff3f745515a0e71dc7dc5839eb483b177e84fd82536b1b2d78b3dde3bb794c3f4f
-  data.tar.gz: 1f82684f46cea8a2ffc0a91dfe503c1d255c3c61c0cd44c6fa4d03c025c5373308cfd93bee258163630d6e77a0b1a7fde9be72a4c6e956b29037796d499dac7a
+  metadata.gz: 1a3bce72d9bd0f1b5f9aa1f92bfca51385163be1b669fee56ef7faf71e0ea45919b3573f056bfb81ff07d8176e095f1c61c8dd0e067186573004686ad6fb05e1
+  data.tar.gz: 3ebfdc6cd512712555ad8f329cf2d8d5f3a1655618cc95b5db5f2b75fdec1f8cb2d0e547252410eb7217a0b5eff6b08f81469983094dcd1fee9a8aa0f95f2c4c

data/app/components/marty/api_config_view.rb

@@ -13,7 +13,9 @@ class Marty::ApiConfigView < Marty::Grid
       :node,
       :attr,
       :logged,
-      :validated,
+      :input_validated,
+      :output_validated,
+      :strict_validate,
       :created_at,
       :updated_at,
     ]
@@ -47,11 +49,11 @@ class Marty::ApiConfigView < Marty::Grid
       c.setter = lambda { |r, v| r.send("#{a}=", v.blank? ? nil : v) }
     end
   end
-  attribute :logged do |c|
-    c.width = 100
-  end
-  attribute :validated do |c|
-    c.width = 110
+  [:logged, :input_validated, :output_validated, :strict_validate].each do
+    |a|
+    attribute a do |c|
+      c.width = 110
+    end
   end
   [:created_at, :updated_at].each do |a|
     attribute a do |c|
@@ -64,7 +66,9 @@ class Marty::ApiConfigView < Marty::Grid
       :node,
       :attr,
       :logged,
-      :validated,
+      :input_validated,
+      :output_validated,
+      :strict_validate,
     ]
   end
 

data/app/controllers/marty/rpc_controller.rb

@@ -28,7 +28,7 @@ class Marty::RpcController < ActionController::Base
       attrs.zip(result)
     rescue => e
       use_message = e.message == 'No such script' ?
-                    'Schema not defined' : 'Problem with schema'
+                    'Schema not defined' : 'Problem with schema: ' + e.message
       raise "Schema error for #{sname}/#{node} "\
             "attrs=#{attrs.join(',')}: #{use_message}"
     end
@@ -64,32 +64,36 @@ class Marty::RpcController < ActionController::Base
 
     return {error: "Malformed params"} unless params.is_a?(Hash)
 
-    need_validate, need_log = [], []
+    need_input_validate, need_output_validate, strict_validate, need_log =
+                                                                [], [], [], []
     Marty::ApiConfig.multi_lookup(sname, node, attrs).each do
-      |attr, log, validate, id|
-      need_validate << attr if validate
+      |attr, log, input_validate, output_validate, strict, id|
+      need_input_validate << attr if input_validate
+      need_output_validate << attr + "_" if output_validate
+      strict_validate << attr if strict
       need_log << id if log
     end
 
+    opt = { :validate_schema    => true,
+                :errors_as_objects  => true,
+                :version            => Marty::JsonSchema::RAW_URI }
+    to_append = {"\$schema" => Marty::JsonSchema::RAW_URI}
+
     validation_error = {}
     err_count = 0
-    if need_validate.present?
+    if need_input_validate.present?
       begin
-        schemas = get_schemas(tag, sname, node, need_validate)
+        schemas = get_schemas(tag, sname, node, need_input_validate)
       rescue => e
         return {error: e.message}
       end
-      opt = { :validate_schema    => true,
-              :errors_as_objects  => true,
-              :version            => Marty::JsonSchema::RAW_URI }
-      to_append = {"\$schema" => Marty::JsonSchema::RAW_URI}
       schemas.each do |attr, sch|
         begin
           er = JSON::Validator.fully_validate(sch.merge(to_append), params, opt)
         rescue NameError
           return {error: "Unrecognized PgEnum for attribute #{attr}"}
         rescue => ex
-          return {error: ex.message}
+          return {error: "#{attr}: #{ex.message}"}
         end
         validation_error[attr] = er.map{ |e| e[:message] } if er.size > 0
         err_count += er.size
@@ -116,8 +120,42 @@ class Marty::RpcController < ActionController::Base
         result = engine.background_eval(node, params, attrs)
         return retval = {"job_id" => result.__promise__.id}
       end
-
       res = engine.evaluate_attrs(node, attrs, params)
+
+      validation_error = {}
+      err_count = 0
+      if need_output_validate.present?
+        begin
+          schemas = get_schemas(tag, sname, node, need_output_validate)
+        rescue => e
+          return {error: e.message}
+        end
+        pairs = attrs.zip(res)
+        pairs.zip(schemas).each do |(attr, res), (_, sch)|
+          begin
+            er = JSON::Validator.fully_validate(sch.merge(to_append), res, opt)
+          rescue NameError
+            return {error: "Unrecognized PgEnum for attribute #{attr}"}
+          rescue => ex
+            return {error: "#{attr}: #{ex.message}"}
+          end
+          validation_error[attr] = er.map{ |e| e[:message] } if er.size > 0
+          err_count += er.size
+        end
+        if err_count > 0
+          res = pairs.map do |attr, res|
+            is_strict = strict_validate.include?(attr)
+            the_error = validation_error[attr]
+
+            Marty::Logger.error("API #{sname}:#{node}.#{attr}",
+                                {error:  the_error,
+                                 data: res}) if the_error
+            is_strict && the_error ?
+              {error: "Error(s) validating: #{the_error}"} : res
+          end
+        end
+      end
+
       return retval = (attrs_atom ? res.first : res)
     rescue => exc
       err_msg = Delorean::Engine.grok_runtime_exception(exc).symbolize_keys

data/app/models/marty/api_config.rb

@@ -6,7 +6,8 @@ class Marty::ApiConfig < Marty::Base
            "AND (attr IS NULL OR attr = ?)",
            script, node, attr]).
            order('node nulls last, attr nulls last').
-           pluck(:logged, :validated, :id)
+           pluck(:logged, :input_validated, :output_validated, :strict_validate,
+                 :id)
     res.first
   end
   def self.multi_lookup(script, node, attrs)

data/db/migrate/302_add_api_configs_validate_result.rb

@@ -0,0 +1,9 @@
+class AddApiConfigsValidateResult < ActiveRecord::Migration
+  def change
+    rename_column :marty_api_configs, :validated, :input_validated
+    add_column :marty_api_configs, :output_validated, :boolean, null: false,
+               default: false
+    add_column :marty_api_configs, :strict_validate, :boolean, null: false,
+               default: false
+  end
+end

data/lib/marty/version.rb

@@ -1,3 +1,3 @@
 module Marty
-  VERSION = "1.0.35"
+  VERSION = "1.0.36"
 end

data/spec/controllers/rpc_controller_spec.rb

@@ -47,7 +47,7 @@ A: M3::A
     g = e * 5 + f
     h = f + 1
     ptest = p * 10
-    result = [{"a": 123, "b": 456}, {"a": 789, "b": 101112}]
+    result = [{"a": p, "b": 456}, {"a": 789, "b": p}]
 eof
 
 sample_script5 = <<eof
@@ -60,6 +60,8 @@ A:
         else if f == "Orange"
         then 2
         else 9
+    result =  [{"a": "str", "b": 456}, {"a": 789, "b": "str"}]
+    result2 = [{"a": "str", "b": 456}, {"a": 789, "b": "str"}]
 eof
 
 sample_script6 = <<eof
@@ -80,6 +82,13 @@ A:
     res = 123
 eof
 
+sample_script9 = <<eof
+A:
+    b =?
+    res = b + 1
+    result = [{"a": 1, "b": res}, {"a": 789, "b": res}]
+eof
+
 script3_schema = <<eof
 A:
     pc = { "properties : {
@@ -94,6 +103,7 @@ A:
             "p" : { "type" : "integer" },
                 }
             }
+    d_ = { "type" : "integer" }
 
     g = { "properties" : {
                   "e" : { "type" : "integer" },
@@ -101,10 +111,29 @@ A:
                 }
           }
 
+    g_ = { "type" : "integer" }
+
     lc = { "properties" : {
                   "p" : { "type" : "integer" },
                 }
             }
+
+    result = { "properties" : {
+            "p" : { "type" : "integer" },
+                }
+             }
+
+    result_ = {
+                 "type": "array",
+                 "minItems": 1,
+                 "items": {
+                   "type": "object",
+                   "properties": {
+                       "a": { "type" : "integer" },
+                       "b": { "type" : "integer" }
+                }
+              }
+          }
 eof
 
 script5_schema = <<eof
@@ -113,6 +142,39 @@ A:
             "f" : { "pg_enum" : "FruitsEnum" },
                 }
             }
+
+    result = { "properties" : {
+            "f" : { "pg_enum" : "FruitsEnum" },
+                }
+            }
+
+    result_ = { "type": "array",
+                 "minItems": 1,
+                 "items": {
+                   "type": "object",
+                   "properties": {
+                       "a": { "type" : "integer" },
+                       "b": { "type" : "string" }
+                   }
+                }
+             }
+
+    result2 = { "properties" : {
+            "f" : { "pg_enum" : "FruitsEnum" },
+                }
+            }
+
+    result2_ = { "type": "array",
+                 "minItems": 1,
+                 "items": {
+                   "type": "object",
+                   "properties": {
+                       "a": { "type" : "integer" },
+                       "b": { "type" : "string" }
+                   }
+                }
+             }
+
 eof
 
 script6_schema = <<eof
@@ -139,6 +201,31 @@ A:
             }
 eof
 
+script9_schema = <<eof
+A:
+    res = { "properties" : {
+            "b" : { "type" : "number" },
+                }
+            }
+
+    result = { "properties" : {
+            "b" : { "type" : "number" },
+                }
+            }
+
+    result_ = {  "type": "array",
+                 "minItems": 1,
+                 "items": {
+                   "type": "object",
+                   "properties": {
+                       "a": { "type" : "integer" },
+                       "b": { "type" : "integer" },
+                       "c": { "type" : "string" }
+                   },
+                   "required" : ["a", "b", "c"]
+                }
+          }
+eof
 
 describe Marty::RpcController do
   before(:each) {
@@ -162,12 +249,14 @@ describe Marty::RpcController do
                          "M6" => sample_script6,
                          "M7" => sample_script7,
                          "M8" => sample_script8,
+                         "M9" => sample_script9,
                          "M3Schemas" => script3_schema,
                          "M4Schemas" => script4_schema,
                          "M5Schemas" => script5_schema,
                          "M6Schemas" => script6_schema,
                          "M7Schemas" => script7_schema,
                          "M8Schemas" => script8_schema,
+                         "M9Schemas" => script9_schema,
                        }, Date.today + 1.minute)
 
     @p1 = Marty::Posting.do_create("BASE", Date.today + 2.minute, 'a comment')
@@ -415,7 +504,7 @@ describe Marty::RpcController do
       attrs: ["result"].to_json,
     }
     # puts 'Z'*40, request.inspect
-    expect(response.body).to eq("a,b\r\n123,456\r\n789,101112\r\n")
+    expect(response.body).to eq("a,b\r\n10,456\r\n789,10\r\n")
   end
 
   it "returns an error message on missing schema script (csv)" do
@@ -423,7 +512,7 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true)
     attrs = ["b"].to_json
     params = {"a" => 5}.to_json
     get 'evaluate', {
@@ -442,7 +531,7 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true)
     attrs = ["b"].to_json
     params = {"a" => 5}.to_json
     get 'evaluate', {
@@ -464,7 +553,7 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true)
     attrs = ["h"].to_json
     params = {"f" => 5}.to_json
     get 'evaluate', {
@@ -474,8 +563,8 @@ describe Marty::RpcController do
       attrs: attrs,
       params: params
     }
-    expect = "Schema error for M4/A attrs=h: Problem with schema\r\n"
-    expect(response.body).to eq("error,#{expect}")
+    expect = "Schema error for M4/A attrs=h: Problem with schema"
+    expect(response.body).to include("error,#{expect}")
   end
 
   it "returns an error message on invalid schema" do
@@ -483,7 +572,7 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true)
     attrs = ["pc"].to_json
     params = {"p" => 5}.to_json
     get 'evaluate', {
@@ -493,7 +582,8 @@ describe Marty::RpcController do
       attrs: attrs,
       params: params
     }
-    expect = "Schema error for M3/A attrs=pc: Problem with schema\r\n"
+    expect = "Schema error for M3/A attrs=pc: Problem with schema: "\
+             "syntax error M3Schemas:2\r\n"
     expect(response.body).to eq("error,#{expect}")
   end
 
@@ -502,7 +592,7 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true)
     attrs = ["d"].to_json
     params = {"p" => "132"}.to_json
     get 'evaluate', {
@@ -522,7 +612,8 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true,
+                             output_validated: true)
     attrs = ["d", "g"].to_json
     params = {"p" => "132", "e" => "55", "f"=>"16"}.to_json
     get 'evaluate', {
@@ -531,7 +622,7 @@ describe Marty::RpcController do
       node: "A",
       attrs: attrs,
       params: params
-    }
+        }
     expect = '""d""=>[""The property \'#/p\' of type string did not '\
              'match the following type: integer'
     expect(response.body).to include(expect)
@@ -543,12 +634,125 @@ describe Marty::RpcController do
     expect(response.body).to include(expect)
   end
 
+  context "output_validation" do
+    before(:all) do
+      @db =  SQLite3::Database.new(Marty::Log.logfile)
+    end
+    before(:each) do
+      @logid = @db.execute('select max(id) from log').first.first || 0 rescue 0
+    end
+    after(:all) do
+      @db.close
+    end
+    it "validates output" do
+      Marty::ApiConfig.create!(script: "M4",
+                               node: "A",
+                               attr: nil,
+                               logged: false,
+                               input_validated: true,
+                               output_validated: true)
+      attrs = ["d", "g", "result"].to_json
+      params = {"p" => 132, "e" => 55, "f"=>16}.to_json
+      get 'evaluate', {
+            format: :json,
+            script: "M4",
+            node: "A",
+            attrs: attrs,
+            params: params
+          }
+      res_hash = JSON.parse(response.body)
+      expect(res_hash).to eq([135,291,[{"a"=>132,"b"=>456},
+                                       {"a"=>789,"b"=>132}]])
+      logs = Marty::Log.where("id > #{@logid}").to_a
+      expect(logs.count).to eq(0)
+    end
+
+    it "validates output (bad type, with strict/non strict errors)" do
+      Marty::ApiConfig.create!(script: "M5",
+                               node: "A",
+                               attr: nil,
+                               logged: false,
+                               input_validated: true,
+                               output_validated: true,
+                               strict_validate: true)
+      Marty::ApiConfig.create!(script: "M5",
+                               node: "A",
+                               attr: "result2",
+                               logged: false,
+                               input_validated: true,
+                               output_validated: true,
+                               strict_validate: false)
+      attrs = ["result", "result2"].to_json
+      params = {"f" => "Banana"}.to_json
+      get 'evaluate', {
+            format: :json,
+            script: "M5",
+            node: "A",
+            attrs: attrs,
+            params: params
+          }
+      res_hash = JSON.parse(response.body)
+      expect(res_hash[0]).to include("error")
+      expect1 = "The property '#/0/b' of type integer did not match the "\
+               "following type: string"
+      expect2 = "The property '#/0/a' of type string did not match the "\
+                "following type: integer"
+      expect(res_hash[0]["error"]).to include(expect1)
+      expect(res_hash[0]["error"]).to include(expect2)
+
+      logs = Marty::Log.where("id > #{@logid}").to_a
+      expect(logs.count).to eq(2)
+      expect(logs[0].message).to eq("API M5:A.result")
+      expect(logs[1].message).to eq("API M5:A.result2")
+      logs.each do |ml|
+        expect(ml.details).to include(expect1)
+        expect(ml.details).to include(expect2)
+        expect(ml.details).to include(
+         ":data=>[{\"a\"=>\"str\", \"b\"=>456}, {\"a\"=>789, \"b\"=>\"str\"}]}")
+      end
+    end
+
+    it "validates output (missing item)" do
+      logid = @db.execute('select max(id) from log').first.first rescue 0
+      Marty::ApiConfig.create!(script: "M9",
+                               node: "A",
+                               attr: nil,
+                               logged: false,
+                               input_validated: true,
+                               output_validated: true,
+                               strict_validate: true)
+      attrs = ["result"].to_json
+      params = {"b" => 122}.to_json
+      get 'evaluate', {
+            format: :json,
+            script: "M9",
+            node: "A",
+            attrs: attrs,
+            params: params
+          }
+      res_hash = JSON.parse(response.body)
+      expect(res_hash[0]).to include("error")
+      expect1 = "The property '#/0' did not contain a required property of 'c'"
+      expect2 = "The property '#/1' did not contain a required property of 'c'"
+      expect(res_hash[0]["error"]).to include(expect1)
+      expect(res_hash[0]["error"]).to include(expect2)
+
+      logs = Marty::Log.where("id > #{@logid}").to_a
+      expect(logs.count).to eq(1)
+      expect(logs[0].message).to eq("API M9:A.result")
+      expect(logs[0].details).to include(expect1)
+      expect(logs[0].details).to include(expect2)
+      expect(logs[0].details).to include(
+         ":data=>[{\"a\"=>1, \"b\"=>123}, {\"a\"=>789, \"b\"=>123}]")
+    end
+  end
+
   it "validates schema" do
     Marty::ApiConfig.create!(script: "M4",
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true)
     attrs = ["lc"].to_json
     params = {"p" => 5}.to_json
     get 'evaluate', {
@@ -566,7 +770,7 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true)
     attrs = ["res"].to_json
     params = {"b" => 5.22}.to_json
     get 'evaluate', {
@@ -576,7 +780,7 @@ describe Marty::RpcController do
       attrs: attrs,
       params: params
     }
-    expect = 'The property \'#/properties/b/type\' of type string '\
+    expect = 'res: The property \'#/properties/b/type\' of type string '\
              'did not match one or more of the required schemas'
     res_hsh = JSON.parse(response.body)
     expect(res_hsh.keys.size).to eq(1)
@@ -594,7 +798,7 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true)
     attrs = ["res"].to_json
     params = {"f" => "Banana"}.to_json
     get 'evaluate', {
@@ -612,7 +816,7 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true)
     attrs = ["res"].to_json
     params = {"f" => "Beans"}.to_json
     get 'evaluate', {
@@ -631,7 +835,7 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true)
     attrs = ["res"].to_json
     params = {"b" => "MemberOfANonExistantEnum"}.to_json
     get 'evaluate', {
@@ -653,7 +857,7 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: false,
-                             validated: true)
+                             input_validated: true)
     skip "pending until a solution is found that handles "\
          "autoload issues involving constantize"
     attrs = ["res"].to_json

data/spec/dummy/public/images/icons

@@ -1 +1 @@
-../extjs/icons/
+spec/dummy/public/images/../extjs/icons/

data/spec/dummy/spec/features/javascripts

@@ -1 +1 @@
-../../../features/javascripts/
+spec/dummy/spec/features/../../../features/javascripts/

data/spec/features/log_view_spec.rb

@@ -18,6 +18,7 @@ feature 'logger view', js: true, capybara: true do
   before(:all) do
     self.use_transactional_fixtures = false
     @db =  SQLite3::Database.new(Marty::Log.logfile)
+    @db.execute("delete from log")
 
     info_s = { info: 'message' }
     error_s = [1, 2, 3, { error: 'message' }]

data/spec/lib/logger_spec.rb

@@ -8,9 +8,9 @@ module Marty
     end
     before(:each) do
       @db =  SQLite3::Database.new(Marty::Log.logfile)
+      @db.execute "delete from log"
     end
     after(:each) do
-      @db.execute "delete from log"
       @db.close
     end
     after(:all) do
@@ -104,6 +104,7 @@ module Marty
       save_clean_db(@clean_file)
       # transactional fixtures interfere with queueing jobs
       self.use_transactional_fixtures = false
+      Marty::Log.cleanup(0)
 
       # Needed here because shutting transactional fixtures off
       # means we lose the globally set user

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: marty
 version: !ruby/object:Gem::Version
-  version: 1.0.35
+  version: 1.0.36
 platform: ruby
 authors:
 - Arman Bostani
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-29 00:00:00.000000000 Z
+date: 2017-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -475,6 +475,7 @@ files:
 - db/migrate/202_add_completion_status_to_event.rb
 - db/migrate/300_create_marty_api_configs.rb
 - db/migrate/301_create_marty_api_log.rb
+- db/migrate/302_add_api_configs_validate_result.rb
 - db/seeds.rb
 - delorean/script_report.dl
 - gemini_deprecations.md
@@ -1650,7 +1651,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: A framework for working with versioned data