marty 1.0.27 → 1.0.28

data/db/migrate/301_create_marty_api_log.rb

@@ -0,0 +1,16 @@
+class CreateMartyApiLog < ActiveRecord::Migration
+  def change
+    create_table :marty_api_logs do |t|
+      t.string :script, null: false
+      t.string :node, null: false
+      t.string :attrs, null: false
+      t.json :input, null: true
+      t.json :output, null: true
+      t.datetime :start_time, null: false
+      t.datetime :end_time, null: false
+      t.string :error, null: true
+      t.string :remote_ip, null: false
+      t.string :auth_name, null: true
+    end
+  end
+end

data/lib/marty.rb

@@ -14,6 +14,7 @@ require 'marty/mcfly_query'
 require 'marty/monkey'
 require 'marty/promise_job'
 require 'marty/lazy_column_loader'
+require 'marty/json_schema'
 
 # This does not get loaded in via bundler unless it is included in the
 # application's Gemfile. Requiring it here removes the need to add it

data/lib/marty/json_schema.rb

@@ -0,0 +1,52 @@
+require 'json-schema'
+
+module Marty
+
+  private
+  class PgEnumAttribute < JSON::Schema::Attribute
+    def self.validate(curr_schema, data, frag, pro, validator, opt={})
+      enum = nil
+      begin
+        enum = curr_schema.schema["pg_enum"].constantize
+      rescue
+        msg = "#{self.class.name} error: '#{data}' is not a pg_enum class"
+        validation_error(pro, msg, frag, curr_schema, self, opt[:record_errors])
+      end
+      if !enum::VALUES.include?(data)
+        msg = "#{self.class.name} error: '#{data}' not contained in #{enum}"
+        validation_error(pro, msg, frag, curr_schema, self, opt[:record_errors])
+      end
+    end
+  end
+
+  class DateTimeFormatAttribute < JSON::Schema::Attribute
+    def self.validate(curr_schema, data, frag, processor, validator, options={})
+      begin
+        DateTime.parse(data).in_time_zone(Rails.configuration.time_zone)
+      rescue
+        msg = "#{self.class.name} error: Can't parse '#{data}' into a DateTime"
+        validation_error( processor,
+                          msg,
+                          frag,
+                          curr_schema,
+                          self,
+                          options[:record_errors])
+      end
+    end
+  end
+
+  class JsonSchema < JSON::Schema::Draft4
+    RAW_URI = "http://json-schema.org/marty-draft/schema#"
+
+    def initialize
+      super
+      @attributes["pg_enum"] = PgEnumAttribute
+      @attributes["datetime_format"] = DateTimeFormatAttribute
+      @uri = JSON::Util::URI.parse(RAW_URI)
+      @names = ["marty-draft", RAW_URI]
+    end
+
+    JSON::Validator.register_validator(self.new)
+  end
+
+end

data/lib/marty/monkey.rb

@@ -209,6 +209,21 @@ class StringEnum < String
   end
   delorean_instance_method :name
   delorean_instance_method :id
+
+  def to_yaml(opts = {})
+    YAML::quick_emit(nil, opts) do |out|
+      out.scalar('stringEnum', self.to_s, :plain)
+    end
+  end
+
+  def marshal_dump
+    self.to_s
+  end
+
+end
+YAML::add_domain_type("pennymac.com,2017-06-02", "stringEnum") do
+  |type, val|
+  StringEnum.new(val)
 end
 
 ######################################################################

data/lib/marty/version.rb

@@ -1,3 +1,3 @@
 module Marty
-  VERSION = "1.0.27"
+  VERSION = "1.0.28"
 end

data/marty.gemspec

@@ -41,4 +41,5 @@ Gem::Specification.new do |s|
   s.add_dependency 'net-ldap', '~> 0.12.0'
   s.add_dependency 'rubyzip'
   s.add_dependency 'sqlite3'
+  s.add_dependency 'json-schema'
 end

data/spec/controllers/rpc_controller_spec.rb

@@ -42,9 +42,62 @@ A: M3::A
     p =? 10
     c = a * 2
     d = pc - 1
+    e =?
+    f =?
+    g = e * 5 + f
+    h = f + 1
+    ptest = p * 10
     result = [{"a": 123, "b": 456}, {"a": 789, "b": 101112}]
 eof
 
+sample_script5 = <<eof
+A:
+    f =?
+    res = if f == "Apple"
+        then 0
+        else if f == "Banana"
+        then 1
+        else if f == "Orange"
+        then 2
+        else 9
+eof
+
+script3_schema = <<eof
+A:
+    pc = { "properties : {
+                  "p" : { "type" : "integer" },
+                }
+            }
+eof
+
+script4_schema = <<eof
+A:
+    d = { "properties" : {
+            "p" : { "type" : "integer" },
+                }
+            }
+
+    g = { "properties" : {
+                  "e" : { "type" : "integer" },
+                  "f" : { "type" : "integer" },
+                }
+          }
+
+    lc = { "properties" : {
+                  "p" : { "type" : "integer" },
+                }
+            }
+eof
+
+script5_schema = <<eof
+A:
+    res = { "properties" : {
+            "f" : { "pg_enum" : "FruitsEnum" },
+                }
+            }
+eof
+
+
 describe Marty::RpcController do
   before(:each) {
     @routes = Marty::Engine.routes
@@ -63,6 +116,10 @@ describe Marty::RpcController do
                          "M2" => sample_script.gsub(/a/, "aa").gsub(/b/, "bb"),
                          "M3" => sample_script3,
                          "M4" => sample_script4,
+                         "M5" => sample_script5,
+                         "M3Schemas" => script3_schema,
+                         "M4Schemas" => script4_schema,
+                         "M5Schemas" => script5_schema,
                        }, Date.today + 1.minute)
 
     @p1 = Marty::Posting.do_create("BASE", Date.today + 2.minute, 'a comment')
@@ -118,6 +175,28 @@ describe Marty::RpcController do
     Delayed::Worker.delay_jobs = true
   end
 
+  it "should be able to post background job with non-array attrs" do
+    Delayed::Worker.delay_jobs = false
+    post 'evaluate', {
+           format: :json,
+           script: "M1",
+           node: "B",
+           attrs: "e",
+           tag: t1.name,
+           params: { a: 333, d: 5}.to_json,
+           background: true,
+         }
+    res = ActiveSupport::JSON.decode response.body
+    expect(res).to include('job_id')
+    job_id = res['job_id']
+
+    promise = Marty::Promise.find_by_id(job_id)
+
+    expect(promise.result).to eq({"e"=>4})
+
+    Delayed::Worker.delay_jobs = true
+  end
+
   it "should be able to post with complex data" do
     post 'evaluate', {
            format: :json,
@@ -291,6 +370,258 @@ describe Marty::RpcController do
     expect(response.body).to eq("a,b\r\n123,456\r\n789,101112\r\n")
   end
 
+  it "returns an error message on missing schema script" do
+    Marty::ApiConfig.create!(script: "M1",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    attrs = ["b"].to_json
+    params = {"a" => 5}.to_json
+    get 'evaluate', {
+      format: :csv,
+      script: "M1",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+    expect = "Schema error for M1/A attrs=b: Schema not defined\r\n"
+    expect(response.body).to eq("error,#{expect}")
+  end
+
+  it "returns an error message on missing attributes in schema script" do
+    Marty::ApiConfig.create!(script: "M4",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    attrs = ["h"].to_json
+    params = {"f" => 5}.to_json
+    get 'evaluate', {
+      format: :csv,
+      script: "M4",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+    expect = "Schema error for M4/A attrs=h: Problem with schema\r\n"
+    expect(response.body).to eq("error,#{expect}")
+  end
+
+  it "returns an error message on invalid schema" do
+    Marty::ApiConfig.create!(script: "M3",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    attrs = ["pc"].to_json
+    params = {"p" => 5}.to_json
+    get 'evaluate', {
+      format: :csv,
+      script: "M3",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+    expect = "Schema error for M3/A attrs=pc: Problem with schema\r\n"
+    expect(response.body).to eq("error,#{expect}")
+  end
+
+  it "returns a validation error when validating a single attribute" do
+    Marty::ApiConfig.create!(script: "M4",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    attrs = ["d"].to_json
+    params = {"p" => "132"}.to_json
+    get 'evaluate', {
+      format: :csv,
+      script: "M4",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+    expect = '""d""=>[""The property \'#/p\' of type string did not '\
+             'match the following type: integer'
+    expect(response.body).to include(expect)
+  end
+
+  it "returns a validation error when validating multiple attributes" do
+    Marty::ApiConfig.create!(script: "M4",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    attrs = ["d", "g"].to_json
+    params = {"p" => "132", "e" => "55", "f"=>"16"}.to_json
+    get 'evaluate', {
+      format: :csv,
+      script: "M4",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+    expect = '""d""=>[""The property \'#/p\' of type string did not '\
+             'match the following type: integer'
+    expect(response.body).to include(expect)
+    expect = '""g""=>[""The property \'#/e\' of type string did not '\
+             'match the following type: integer'
+    expect(response.body).to include(expect)
+    expect = 'The property \'#/f\' of type string did not '\
+             'match the following type: integer'
+    expect(response.body).to include(expect)
+  end
+
+  it "validates schema" do
+    Marty::ApiConfig.create!(script: "M4",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    attrs = ["lc"].to_json
+    params = {"p" => 5}.to_json
+    get 'evaluate', {
+      format: :csv,
+      script: "M4",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+    expect(response.body).to eq("9\r\n9\r\n")
+  end
+
+  class FruitsEnum
+    VALUES=Set['Apple', 'Banana', 'Orange']
+  end
+
+  it "validates schema with a pg_enum (Positive)" do
+    Marty::ApiConfig.create!(script: "M5",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    attrs = ["res"].to_json
+    params = {"f" => "Banana"}.to_json
+    get 'evaluate', {
+      format: :csv,
+      script: "M5",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+    expect(response.body).to eq("1\r\n")
+  end
+
+  it "validates schema with a pg_enum (Negative)" do
+    Marty::ApiConfig.create!(script: "M5",
+                             node: "A",
+                             attr: nil,
+                             logged: false,
+                             validated: true)
+    attrs = ["res"].to_json
+    params = {"f" => "Beans"}.to_json
+    get 'evaluate', {
+      format: :csv,
+      script: "M5",
+      node: "A",
+      attrs: attrs,
+      params: params
+    }
+    expect = '""res""=>[""Class error: \'Beans\' not contained in FruitsEnum'
+    expect(response.body).to include(expect)
+  end
+
+  it "should log good req" do
+    Marty::ApiConfig.create!(script: "M3",
+                             node: "A",
+                             attr: nil,
+                             logged: true)
+    attrs = ["lc"].to_json
+    params = {"p" => 5}
+    get 'evaluate', {
+      format: :csv,
+      script: "M3",
+      node: "A",
+      attrs: attrs,
+      params: params.to_json
+    }
+    expect(response.body).to eq("9\r\n9\r\n")
+    log = Marty::ApiLog.order(id: :desc).first
+
+    expect(log.script).to eq("M3")
+    expect(log.node).to eq("A")
+    expect(log.attrs).to eq(attrs)
+    expect(log.input).to eq(params)
+    expect(log.output).to eq([[9, 9]])
+    expect(log.remote_ip).to eq("0.0.0.0")
+    expect(log.error).to eq(nil)
+
+  end
+
+  it "should log good req [background]" do
+    Marty::ApiConfig.create!(script: "M3",
+                             node: "A",
+                             attr: nil,
+                             logged: true)
+    attrs = ["lc"].to_json
+    params = {"p" => 5}
+    get 'evaluate', {
+      format: :csv,
+      script: "M3",
+      node: "A",
+      attrs: attrs,
+      params: params.to_json,
+      background: true
+    }
+    expect(response.body).to match(/job_id,/)
+    log = Marty::ApiLog.order(id: :desc).first
+
+    expect(log.script).to eq("M3")
+    expect(log.node).to eq("A")
+    expect(log.attrs).to eq(attrs)
+    expect(log.input).to eq(params)
+    expect(log.output).to include("job_id")
+    expect(log.remote_ip).to eq("0.0.0.0")
+    expect(log.error).to eq(nil)
+
+  end
+
+  it "should not log if it should not log" do
+    get 'evaluate', {
+      format: :json,
+      script: "M1",
+      node: "A",
+      attrs: ["a", "b"].to_json,
+      tag: t1.name,
+    }
+    expect(Marty::ApiLog.count).to eq(0)
+  end
+
+  it "should handle atom attribute" do
+    Marty::ApiConfig.create!(script: "M3",
+                             node: "A",
+                             attr: nil,
+                             logged: true)
+    params = {"p" => 5}
+    get 'evaluate', {
+      format: :csv,
+      script: "M3",
+      node: "A",
+      attrs: "lc",
+      params: params.to_json
+    }
+    expect(response.body).to eq("9\r\n9\r\n")
+    log = Marty::ApiLog.order(id: :desc).first
+    expect(log.script).to eq("M3")
+    expect(log.node).to eq("A")
+    expect(log.attrs).to eq("lc")
+    expect(log.input).to eq(params)
+    expect(log.output).to eq([9, 9])
+    expect(log.remote_ip).to eq("0.0.0.0")
+    expect(log.error).to eq(nil)
+  end
+
   it "should support api authorization - api_key not required" do
     api = Marty::ApiAuth.new
     api.app_name = 'TestApp'
@@ -327,6 +658,9 @@ describe Marty::RpcController do
     api.script_name = 'M3'
     api.save!
 
+    apic = Marty::ApiConfig.create!(script: 'M3',
+                                    logged: true)
+
     get 'evaluate', {
       format: :json,
       script: "M3",
@@ -335,6 +669,13 @@ describe Marty::RpcController do
       api_key: api.api_key,
     }
     expect(response.body).to eq([7].to_json)
+    log = Marty::ApiLog.order(id: :desc).first
+    expect(log.script).to eq('M3')
+    expect(log.node).to eq('C')
+    expect(log.attrs).to eq(%Q!["pc"]!)
+    expect(log.output).to eq([7])
+    expect(log.remote_ip).to eq("0.0.0.0")
+    expect(log.auth_name).to eq("TestApp")
   end
 
   it "should support api authorization - api_key required but incorrect" do
@@ -356,7 +697,7 @@ describe Marty::RpcController do
   context "error handling" do
     it 'returns bad attrs if attrs is not a string' do
       get :evaluate, format: :json, attrs: 0
-      expect(response.body).to match(/"error":"Bad attrs"/)
+      expect(response.body).to match(/"error":"Malformed attrs"/)
     end
 
     it 'returns malformed attrs for improperly formatted json' do