marty 1.0.47 → 1.0.48

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5c800ee99b1eb7023ea3a849088707a0200ba7a4
-  data.tar.gz: cdc3f0083bb23d6f42e9c94cbb6b50f927cf11c5
+  metadata.gz: 376e0b397909c27475e6b4cdae94dee42567e903
+  data.tar.gz: b93e6460252eb76a0391732a33ed14304d422b90
 SHA512:
-  metadata.gz: 1f0f1912b8bcb74509857d1291c1a578960fe061625bc0e087b04eedc04aad4ae466008f6cf4d71a132392c8e3b1b0c88cce1a6bd4665d7e8aec421074f63e50
-  data.tar.gz: 66e079cb3b5034ab20cda7f3e4752d437ce92caa16b617e8362a5e28e7b0dbe1fb883526762d962198fdd07e1005b46e05ce4f833e1762781c615f8d0b140386
+  metadata.gz: 5d573ea332aea15f9eb89406a0158c00c359ab2761ddbc53f07804e7bb2044862ca02a31ab1308588c6a2a5d1901797e18367ec80a81738dce87dd2278585fdc
+  data.tar.gz: 6d1523146cc9ed32261d66676c54d673c4221b23cc915b9005a244da666a1c864a9ec451750138773b0a342715f05a8b90fc242fb7a99f7896fddf606ca82705

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    marty (1.0.46)
+    marty (1.0.48)
       axlsx (= 2.1.0pre)
       coderay
       delorean_lang (~> 0.1)

data/app/components/marty/api_log_view.rb

@@ -3,28 +3,34 @@ class Marty::ApiLogView < Marty::Grid
   has_marty_permissions read: :admin,
                         update: :admin
 
+  DATE_OP_MAP = {
+    'eq' => '=',
+    'gt' => '>',
+    'lt' => '<'
+  }
+
+  @@attrs = [
+    :script,
+    :node,
+    :attrs,
+    :input,
+    :output,
+    :error,
+    :remote_ip,
+    :auth_name,
+    :start_time,
+    :end_time,
+  ]
+
   def configure(c)
     super
     c.editing = :in_form
     c.paging = :buffered
     c.title = 'Api Log View'
-    c.model = Marty::ApiLog
-    c.attributes = [
-      :script,
-      :node,
-      :attrs,
-      :start_time,
-      :end_time,
-      :input,
-      :output,
-      :error,
-      :remote_ip,
-      :auth_name,
-    ]
-
-    c.store_config.merge!(
-      {sorters: [{ property: :start_time, direction: :desc }]},
-    )
+    c.model = Marty::Log
+    c.attributes = @@attrs
+    c.scope = {message_type: 'api'}
+    c.store_config.merge!(sorters: [{property: :timestamp, direction: 'DESC'}])
   end
 
   component :edit_window do |c|
@@ -32,13 +38,6 @@ class Marty::ApiLogView < Marty::Grid
     c.width = 1200
   end
 
-  [:script, :node, :attrs, :remote_ip, :auth_name, :start_time,
-   :end_time].each do |a|
-    attribute a do |c|
-      c.read_only = true
-    end
-  end
-
   def default_form_items
     [
       :script,
@@ -54,20 +53,42 @@ class Marty::ApiLogView < Marty::Grid
     ]
   end
 
-  [:input, :output].each do |a|
+  @@attrs.each do |a|
     attribute a do |c|
-      c.text = a.to_s
-      c.width     = 900
-      c.read_only = true
-      c.type = :string
-      c.getter = lambda { |r| r.send(a).pretty_inspect }
+      c.filterable    = true
+      c.read_only     = true
+      c.getter        = lambda {|r| r.details[a.to_s] }
+      c.sorting_scope = lambda {
+        |r, dir|
+        r.order("details->>'#{a.to_s}'" + dir.to_s)
+      }
+      c.filter_with = lambda {
+        |r, v, op|
+        r.where("details->>'#{a.to_s}' #{op} '#{v}%'")
+      } unless [:start_time, :endtime].include?(a)
+
+      case a
+      when :start_time, :end_time
+        c.type        = :datetime
+        c.format      = 'Y-m-d h:i:s'
+        c.getter      = lambda {|r| Time.zone.parse(r.details[a.to_s])}
+        c.filter_with = lambda {
+          |r, v, op|
+          r.where("(details->>'#{a.to_s}')::date #{DATE_OP_MAP[op]} '#{v}%'")
+        }
+      when :input, :output
+        c.getter    = lambda { |r| r.details[a.to_s].pretty_inspect }
+        c.width     = 900
+        c.read_only = true
+      end
     end
+  end
+
+  [:input, :output].each do |a|
     column a do |c|
-      c.width = 250
-      c.type = :string
-      c.getter = lambda { |r| r.send(a).to_json }
+      c.width  = 250
+      c.getter = lambda { |r| r.details[a.to_s].to_json }
     end
   end
-
 end
 ApiLogView = Marty::ApiLogView

data/app/components/marty/report_form.rb

@@ -14,11 +14,17 @@ class Marty::ReportForm < Marty::Form
     a.disabled = false
   end
 
+  action :link do |a|
+    a.text     = a.tooltip = I18n.t("reporting.link")
+    a.icon     = :link_go
+    a.disabled = false
+  end
+
   ######################################################################
 
   def default_bbar
     [
-      '->', :apply, :foreground
+      '->', :apply, :foreground, :link
     ]
   end
 
@@ -134,6 +140,37 @@ class Marty::ReportForm < Marty::Form
        document.body.removeChild(form);
     }
     JS
+
+    c.netzke_on_link = l(<<-JS)
+    function() {
+       var values = this.getForm().getValues();
+
+       // check for early url generation and exit with error message
+       if (values['selected_script_name'] == null) {
+         alert("Please select a report before generating url.");
+         return;
+       }
+
+       params = {
+         "format": this.repformat,
+         "reptitle": this.reptitle
+       }
+
+       for (var key in values) {if (values[key] == "") {delete values[key]}}
+       data = Ext.encode(values)
+
+       // construct url
+       var proto_host = location.protocol + '//' + location.host
+       var url  = proto_host + '/report?data=' + data
+       for (var key in params) {
+         if (params[key] == "") continue;
+         url += '&' + key + '=' + params[key];
+       }
+       url = encodeURI(url)
+       var win = window.open('');
+       win.document.write(url.link(url));
+     }
+    JS
   end
 
   endpoint :netzke_load do |params|
@@ -231,7 +268,7 @@ class Marty::ReportForm < Marty::Form
     c.reptitle           = title
     c.authenticity_token = controller.send(:form_authenticity_token)
 
-    actions[:foreground].disabled = !!background_only
+    [:foreground, :link].each{|a| actions[a].disabled = !!background_only}
   end
 end
 

data/app/controllers/marty/diagnostic_controller.rb

@@ -83,7 +83,7 @@ module Marty
         display = <<-ERB
                 <% inconsistent = diff(data) %>
                 <h3><%=name.demodulize%></h3>
-                <%='<h3 class="error">&#x26a0; Issues Detected</h3>' if
+                <%='<h3 class="error">Issues Detected</h3>' if
                    inconsistent%>
                 <div class="wrapper">
                 <% data.each do |node, result| %>

data/app/controllers/marty/rpc_controller.rb

@@ -155,10 +155,13 @@ class Marty::RpcController < ActionController::Base
         rescue => e
           return {error: e.message}
         end
+        shash = Hash[schemas]
         pairs = attrs.zip(res)
-        pairs.zip(schemas).each do |(attr, res), (_, sch)|
+        pairs.each do |attr, result|
+          next unless sch = shash[attr+"_"]
+          next if result.is_a?(Hash) && result["error"]
           begin
-            er = JSON::Validator.fully_validate(sch.merge(to_append), res, opt)
+            er = JSON::Validator.fully_validate(sch.merge(to_append), result, opt)
           rescue NameError
             return {error: "Unrecognized PgEnum for attribute #{attr}"}
           rescue => ex
@@ -168,20 +171,19 @@ class Marty::RpcController < ActionController::Base
           err_count += er.size
         end
         if err_count > 0
-          res = pairs.map do |attr, res|
-            is_strict = strict_validate.include?(attr)
-            the_error = validation_error[attr]
+          res = pairs.map do |attr, result|
+            next result unless shash[attr+"_"]
+            next result unless the_error = validation_error[attr]
 
             Marty::Logger.error("API #{sname}:#{node}.#{attr}",
                                 {error:  the_error,
-                                 data: res}) if the_error
-            is_strict && the_error ?
+                                 data: result})
+            strict_validate.include?(attr) ?
               {error: "Error(s) validating: #{the_error}",
-               data: res} : res
+               data: result} : result
           end
         end
       end
-
       return retval = (attrs_atom ? res.first : res)
     rescue => exc
       err_msg = Delorean::Engine.grok_runtime_exception(exc).symbolize_keys
@@ -189,9 +191,9 @@ class Marty::RpcController < ActionController::Base
       return retval = err_msg
     ensure
       error = Hash === retval ? retval[:error] : nil
-
-      # FIXME: how do we know this isn't going to fail??
-      Marty::ApiLog.create!(script:     sname,
+      Marty::Log.write_log('api',
+                           "#{sname} - #{node}",
+                           {script:     sname,
                             node:       node,
                             attrs:      (attrs_atom ? attrs.first : attrs),
                             input:      orig_params,
@@ -200,8 +202,8 @@ class Marty::RpcController < ActionController::Base
                             end_time:   Time.zone.now,
                             error:      error,
                             remote_ip:  request.remote_ip,
-                            auth_name:  auth,
-                           ) if need_log.present?
+                            auth_name:  auth
+                           }) if need_log.present?
     end
   end
 end

data/app/models/marty/data_grid.rb

@@ -152,11 +152,13 @@ class Marty::DataGrid < Marty::Base
   PLV_DT_FMT = "%Y-%m-%d %H:%M:%S.%N6"
 
   def plv_lookup_grid_distinct(h_passed, ret_grid_data=false, distinct=true)
-    row_info = {"id"         => id,
-                "group_id"   => group_id,
-                "created_dt" =>
-              (@@dtcache ||= {})[created_dt] ||= created_dt.strftime(PLV_DT_FMT)
-               }
+    row_info = {
+      "id"         => id,
+      "group_id"   => group_id,
+      "created_dt" => (
+        @@dtcache ||= {})[created_dt] ||= created_dt.strftime(PLV_DT_FMT)
+    }
+
     h = metadata.each_with_object({}) do |m, h|
       attr = m["attr"]
       inc = h_passed.fetch(attr, :__nf__)
@@ -164,19 +166,19 @@ class Marty::DataGrid < Marty::Base
       h[attr] = (defined? inc.name) ? inc.name : inc
     end
 
-    fn = "lookup_grid_distinct"
-    hjson = "'#{h.to_json}'::JSONB"
+    fn     = "lookup_grid_distinct"
+    hjson  = "'#{h.to_json}'::JSONB"
     rijson = "'#{row_info.to_json}'::JSONB"
     params = "#{hjson}, #{rijson}, #{ret_grid_data}, #{distinct}"
-    sql = "SELECT #{fn}(#{params})"
-    raw = ActiveRecord::Base.connection.execute(sql)[0][fn]
-    res = JSON.parse(raw)
+    sql    = "SELECT #{fn}(#{params})"
+    raw    = ActiveRecord::Base.connection.execute(sql)[0][fn]
+    res    = JSON.parse(raw)
 
     if res["error"]
       msg = res["error"]
       parms, sqls, ress, dg = res["error_extra"].values_at(
-                                "params", "sql",
-                                "results", "dg")
+                           "params", "sql", "results", "dg")
+
       raise "DG #{name}: Error in PLV8 call: #{msg}\n"\
             "params: #{parms}\n"\
             "sqls: #{sqls}\n"\

data/app/models/marty/log.rb

@@ -6,9 +6,9 @@ class Marty::Log < Marty::Base
   def self.write_log(type, message, details)
     begin
       create!(message_type: type,
-             message: message,
-             details: details,
-             timestamp: Time.zone.now)
+              message: message,
+              details: details,
+              timestamp: Time.zone.now)
     rescue => e
       Marty::Util.logger.error("Marty::Logger failure: #{e.message}")
     end

data/app/views/marty/diagnostic/op.html.erb

@@ -51,7 +51,7 @@
   </head>
   <body>
   <div style="text-align:center">
-    <h1><%=Rails.application.class.parent_name%> Diagnostic &#x26E8</h1>
+    <h1><%=Rails.application.class.parent_name%> Diagnostic</h1>
     <%== @result %>
   </div>
 </body>

data/config/locales/en.yml

@@ -119,6 +119,7 @@ en:
     select_script: Select Script
     generate: Generate Report
     background: Background Report
+    link: Generate URL
 
   jobs:
     promise_view: Jobs Dashboard

data/config/routes.rb

@@ -1,3 +1,7 @@
+Rails.application.routes.draw do
+  get 'report', to: 'marty/report#index'
+end
+
 Marty::Engine.routes.draw do
   match via: [:get, :post], "rpc/:action(.:format)" => "rpc", as: :rpc
   get "job/:action" => "job", as: :job

data/db/migrate/304_drop_marty_api_logs.rb

@@ -0,0 +1,9 @@
+class DropMartyApiLogs < ActiveRecord::Migration
+  def self.up
+    drop_table :marty_api_logs
+  end
+
+  def self.down
+    announce("No-op on DropMartyApiLogs.down")
+  end
+end

data/lib/marty/data_exporter.rb

@@ -50,6 +50,9 @@ class Marty::DataExporter
     # FIXME: very hacky to default row_sep to CRLF
     conf[:row_sep] ||= "\r\n"
 
+    # remove non CSV.generate options before entering generate blocks
+    readable = conf.delete(:readable)
+
     # FIXME: the following is ridiculously complex. We have different
     # data paths for hashes and arrays.  Also, arrays can turn into
     # hashes is all their items are hashes!  We map to complex objects
@@ -69,7 +72,7 @@ class Marty::DataExporter
         csv << x.map { |v|
           case v
           when Array, Hash
-            encode_json(v.to_json)
+            readable ? v.to_s : encode_json(v.to_json)
           when nil
             nil
           else

data/lib/marty/version.rb

@@ -1,3 +1,3 @@
 module Marty
-  VERSION = "1.0.47"
+  VERSION = "1.0.48"
 end

data/spec/controllers/diagnostic_controller_spec.rb

@@ -56,7 +56,7 @@ module Marty
     def version_display_fail val
       <<-ERB
       <h3>Version</h3>
-      <h3 class="error"> &#x26a0; Issues Detected </h3>
+      <h3 class="error">Issues Detected </h3>
       <div class="wrapper">
       <table>
         <th class="error">node1</th>

data/spec/controllers/rpc_controller_spec.rb

@@ -46,7 +46,9 @@ A: M3::A
     f =?
     g = e * 5 + f
     h = f + 1
+    i =?
     ptest = p * 10
+    ii = i
     result = [{"a": p, "b": 456}, {"a": 789, "b": p}]
 eof
 
@@ -134,6 +136,10 @@ A:
             }
     d_ = { "type" : "integer" }
 
+    ii = {}
+
+    ii_ = { "type" : "integer" }
+
     g = { "properties" : {
                   "e" : { "type" : "integer" },
                   "f" : { "type" : "integer" },
@@ -740,6 +746,35 @@ describe Marty::RpcController do
                                attr: nil,
                                logged: false,
                                input_validated: true,
+                               output_validated: true,
+                               strict_validate: true)
+      attrs = ["d", "g", "ii", "result"].to_json
+      params = {"p" => 132, "e" => 55, "f"=>16, "i"=>"string"}.to_json
+      get 'evaluate', {
+            format: :json,
+            script: "M4",
+            node: "A",
+            attrs: attrs,
+            params: params
+          }
+      res_hash = JSON.parse(response.body)
+      errpart = "of type string did not match the following type: integer"
+      expect(res_hash[0]).to eq(135)
+      expect(res_hash[1]).to eq(291)
+      expect(res_hash[2]["error"]).to include(errpart)
+      expect(res_hash[3]).to eq([{"a"=>132,"b"=>456},
+                                 {"a"=>789,"b"=>132}])
+      logs = Marty::Log.all
+      expect(logs.count).to eq(1)
+      expect(logs[0].details["error"][0]).to include(errpart)
+    end
+    it "validates output 2" do
+      # not all attrs being validated
+      Marty::ApiConfig.create!(script: "M4",
+                               node: "A",
+                               attr: "result",
+                               logged: false,
+                               input_validated: true,
                                output_validated: true)
       attrs = ["d", "g", "result"].to_json
       params = {"p" => 132, "e" => 55, "f"=>16}.to_json
@@ -968,25 +1003,25 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: true)
-    attrs = ["lc"].to_json
+    attrs = ["lc"]
     params = {"p" => 5}
     get 'evaluate', {
       format: :csv,
       script: "M3",
       node: "A",
-      attrs: attrs,
+      attrs: attrs.to_json,
       params: params.to_json
     }
     expect(response.body).to eq("9\r\n9\r\n")
-    log = Marty::ApiLog.order(id: :desc).first
+    log = Marty::Log.order(id: :desc).first
 
-    expect(log.script).to eq("M3")
-    expect(log.node).to eq("A")
-    expect(log.attrs).to eq(attrs)
-    expect(log.input).to eq(params)
-    expect(log.output).to eq([[9, 9]])
-    expect(log.remote_ip).to eq("0.0.0.0")
-    expect(log.error).to eq(nil)
+    expect(log.details['script']).to eq("M3")
+    expect(log.details['node']).to eq("A")
+    expect(log.details['attrs']).to eq(attrs)
+    expect(log.details['input']).to eq(params)
+    expect(log.details['output']).to eq([[9,9]])
+    expect(log.details['remote_ip']).to eq("0.0.0.0")
+    expect(log.details['error']).to eq(nil)
 
   end
 
@@ -995,26 +1030,18 @@ describe Marty::RpcController do
                              node: "A",
                              attr: nil,
                              logged: true)
-    attrs = ["lc"].to_json
+    attrs = ["lc"]
     params = {"p" => 5}
     get 'evaluate', {
       format: :csv,
       script: "M3",
       node: "A",
-      attrs: attrs,
+      attrs: attrs.to_json,
       params: params.to_json,
       background: true
     }
     expect(response.body).to match(/job_id,/)
-    log = Marty::ApiLog.order(id: :desc).first
-
-    expect(log.script).to eq("M3")
-    expect(log.node).to eq("A")
-    expect(log.attrs).to eq(attrs)
-    expect(log.input).to eq(params)
-    expect(log.output).to include("job_id")
-    expect(log.remote_ip).to eq("0.0.0.0")
-    expect(log.error).to eq(nil)
+    log = Marty::Log.order(id: :desc).first
 
   end
 
@@ -1026,7 +1053,7 @@ describe Marty::RpcController do
       attrs: ["a", "b"].to_json,
       tag: t1.name,
     }
-    expect(Marty::ApiLog.count).to eq(0)
+    expect(Marty::Log.count).to eq(0)
   end
 
   it "should handle atom attribute" do
@@ -1043,14 +1070,16 @@ describe Marty::RpcController do
       params: params.to_json
     }
     expect(response.body).to eq("9\r\n9\r\n")
-    log = Marty::ApiLog.order(id: :desc).first
-    expect(log.script).to eq("M3")
-    expect(log.node).to eq("A")
-    expect(log.attrs).to eq("lc")
-    expect(log.input).to eq(params)
-    expect(log.output).to eq([9, 9])
-    expect(log.remote_ip).to eq("0.0.0.0")
-    expect(log.error).to eq(nil)
+    log = Marty::Log.order(id: :desc).first
+
+    expect(log.details['script']).to eq("M3")
+    expect(log.details['node']).to eq("A")
+    expect(log.details['attrs']).to eq("lc")
+    expect(log.details['input']).to eq(params)
+    expect(log.details['output']).to eq([9, 9])
+    expect(log.details['remote_ip']).to eq("0.0.0.0")
+    expect(log.details['error']).to eq(nil)
+
   end
 
   it "should support api authorization - api_key not required" do
@@ -1092,21 +1121,23 @@ describe Marty::RpcController do
     apic = Marty::ApiConfig.create!(script: 'M3',
                                     logged: true)
 
+    attrs = ["pc"]
     get 'evaluate', {
       format: :json,
       script: "M3",
       node: "C",
-      attrs: ["pc"].to_json,
+      attrs: attrs.to_json,
       api_key: api.api_key,
     }
     expect(response.body).to eq([7].to_json)
-    log = Marty::ApiLog.order(id: :desc).first
-    expect(log.script).to eq('M3')
-    expect(log.node).to eq('C')
-    expect(log.attrs).to eq(%Q!["pc"]!)
-    expect(log.output).to eq([7])
-    expect(log.remote_ip).to eq("0.0.0.0")
-    expect(log.auth_name).to eq("TestApp")
+    log = Marty::Log.order(id: :desc).first
+
+    expect(log.details['script']).to eq("M3")
+    expect(log.details['node']).to eq("C")
+    expect(log.details['attrs']).to eq(attrs)
+    expect(log.details['output']).to eq([7])
+    expect(log.details['remote_ip']).to eq("0.0.0.0")
+    expect(log.details['auth_name']).to eq("TestApp")
   end
 
   it "should support api authorization - api_key required but incorrect" do

data/spec/features/reporting_spec.rb

@@ -100,6 +100,16 @@ DELOREAN
     end
   end
 
+  def generate_rep_url format: 'csv'
+    URI.encode("http://#{Capybara.current_session.server.host}"\
+               ":#{Capybara.current_session.server.port}"\
+               '/report?data='\
+               '{"pt_name":"XYZ",'\
+               '"selected_script_name":"SomeReport",'\
+               '"selected_node":"AA",'\
+               "\"selected_testing\":\"true\"}&format=#{format}&reptitle=AA")
+  end
+
   it 'run_script displays proper data' do
     log_in_as('dev1')
     go_to_reporting
@@ -249,4 +259,38 @@ DELOREAN
                                              '---')
     end
   end
+
+  it 'can generate an url that can be used to access report' do
+    log_in_as('dev1')
+    go_to_reporting
+
+    tag_grid = netzke_find('tag_grid')
+    script_grid = netzke_find('script_grid')
+
+    and_by 'select SomeReport script & AA node' do
+      script_grid.select_row(1)
+      select_node('AA (csv)')
+    end
+
+    and_by 'fill form and navigate to generated link' do
+      wait_for_ajax
+      set_field_value('XYZ', '', 'pt_name')
+      set_field_value('true', 'textfield', 'selected_testing')
+      new_window = window_opened_by {press("Generate URL")}
+      within_window new_window do
+        url = generate_rep_url
+        expect(page).to have_content(url)
+      end
+      new_window.close
+    end
+  end
+
+  it 'can generate report through generated url' do
+    url = generate_rep_url(format: 'txt') + URI.encode('&disposition=inline')
+    visit url
+    wait_for_element do
+      expect(page).to have_content('XYZ1,XYZ2,XYZ3,XYZ4 1,2,3,4 2,4,6,8 ' +
+                                   '3,6,9,12 4,8,12,16')
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: marty
 version: !ruby/object:Gem::Version
-  version: 1.0.47
+  version: 1.0.48
 platform: ruby
 authors:
 - Arman Bostani
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-02 00:00:00.000000000 Z
+date: 2017-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -413,7 +413,6 @@ files:
 - app/helpers/marty/script_set.rb
 - app/models/marty/api_auth.rb
 - app/models/marty/api_config.rb
-- app/models/marty/api_log.rb
 - app/models/marty/base.rb
 - app/models/marty/config.rb
 - app/models/marty/data_grid.rb
@@ -481,6 +480,7 @@ files:
 - db/migrate/301_create_marty_api_log.rb
 - db/migrate/302_add_api_configs_validate_result.rb
 - db/migrate/303_create_marty_logs.rb
+- db/migrate/304_drop_marty_api_logs.rb
 - db/migrate/400_create_dg_plv8_v1_fns.rb
 - db/seeds.rb
 - delorean/script_report.dl

data/app/models/marty/api_log.rb

@@ -1,5 +0,0 @@
-class Marty::ApiLog < Marty::Base
-  validates_presence_of :script, :node, :attrs, :start_time, :end_time,
-                        :remote_ip
-
-end