marty 1.0.44 → 1.0.46

data/app/controllers/marty/rpc_controller.rb

@@ -34,6 +34,11 @@ class Marty::RpcController < ActionController::Base
     end
   end
 
+  def massage_message(msg)
+    m = %r|'#/([^']+)' of type ([^ ]+) matched the disallowed schema|.match(msg)
+    return msg unless m
+    "disallowed parameter '#{m[1]}' of type #{m[2]} was received"
+  end
   def _get_errors(errs)
     if errs.is_a?(Array)
       errs.map { |err| _get_errors(err) }
@@ -41,10 +46,11 @@ class Marty::RpcController < ActionController::Base
       if !errs.include?(:failed_attribute)
         errs.map { |k, v| _get_errors(v) }
       else
-        fa, message, errors = errs.values_at(:failed_attribute,
-                                             :message, :errors)
-        (['AllOf','AnyOf','Not'].include?(fa) ? [] : [message]) +
-          _get_errors(errors || {})
+        fa, fragment, message, errors = errs.values_at(:failed_attribute,
+                                                       :fragment,
+                                                       :message, :errors)
+        ((['AllOf','AnyOf','Not'].include?(fa) && fragment =='#/') ?
+           [] : [massage_message(message)]) + _get_errors(errors || {})
       end
     end
   end

data/app/models/marty/data_grid.rb

@@ -1,5 +1,4 @@
 class Marty::DataGrid < Marty::Base
-
   # If data_type is nil, assume float
   DEFAULT_DATA_TYPE = "float"
 
@@ -146,102 +145,52 @@ class Marty::DataGrid < Marty::Base
     data_type.constantize rescue nil
   end
 
-  def query_grid_dir(h, infos)
-    return [0] if infos.empty?
-
-    sqla = infos.map do |inf|
-      type, attr = inf["type"], inf["attr"]
-
-      next unless h.has_key?(attr)
-
-      v = h[attr]
-
-      ix_class = INDEX_MAP[type] || INDEX_MAP["string"]
-
-      q = "key IS NULL"
-
-      unless v.nil?
-        q = case type
-            when "boolean"
-              "key = ?"
-            when "numrange", "int4range"
-              "key @> ?"
-            else # "string", "integer", AR klass
-              "key @> ARRAY[?]"
-            end + " OR #{q}"
-
-        # FIXME: very hacky -- need to cast numrange/intrange values or
-        # we get errors from PG.
-        v = case type
-            when "string"
-              v.to_s
-            when "numrange"
-              v.to_f
-            when "int4range", "integer"
-              v.to_i
-            when "boolean"
-              v
-            else # AR class
-              # FIXME: really hacky to hard-code "name".  Used to
-              # perform to_s which could lead ot strange failures when
-              # model had no to_s defined.
-              begin
-                String === v ? v : v.name
-              rescue NoMethodError
-                raise "could not get name for #{v}"
-              end
-            end
-      end
-
-      # FIXME: could potentially order results by key NULLS LAST.
-      # This would prefer more specific rather than wild card
-      # solutions.  However, would need to figure out how to preserve
-      # ordering on subsequent INTERSECT operations.
-      ix_class.
-        select(:index).
-        distinct.
-        where(data_grid_id: group_id,
-              created_dt:   created_dt,
-              attr:         inf["attr"],
-             ).
-        where(q, v).to_sql
-    end.compact
-
-    sql = sqla.join(" INTERSECT ")
-
-    self.class.connection.execute(sql).to_a.map { |hh| hh["index"].to_i }
+  def self.clear_dtcache
+    @@dtcache = {}
   end
 
-  def lookup_grid_distinct(pt, h, return_grid_data=false, distinct=true)
-    isets = ["h", "v"].each_with_object({}) do |dir, ih|
-      infos = dir_infos(dir)
-
-      ih[dir] = query_grid_dir(h, infos)
+  PLV_DT_FMT = "%Y-%m-%d %H:%M:%S.%N6"
 
-      unless ih[dir] or return_grid_data
-        attrs = infos.map { |inf| inf["attr"] }
-
-        raise "#{dir} attrs not provided: %s" % attrs.join(',')
-      end
-
-      raise "Grid #{name}, (#{ih[dir].count}) #{dir} matches > 1." if
-        distinct && ih[dir] && ih[dir].count > 1
+  def plv_lookup_grid_distinct(h_passed, ret_grid_data=false, distinct=true)
+    row_info = {"id"         => id,
+                "group_id"   => group_id,
+                "created_dt" =>
+              (@@dtcache ||= {})[created_dt] ||= created_dt.strftime(PLV_DT_FMT)
+               }
+    h = metadata.each_with_object({}) do |m, h|
+      attr = m["attr"]
+      inc = h_passed.fetch(attr, :__nf__)
+      next if inc == :__nf__
+      h[attr] = (defined? inc.name) ? inc.name : inc
     end
 
-    # deterministic result: pick min index when there's a choice
-    vi, hi = isets["v"].min, isets["h"].min if isets["v"] && isets["h"]
-
-    raise "DataGrid lookup failed #{name}" unless (vi && hi) or lenient or
-      return_grid_data
-
-    modified_data, modified_metadata = modify_grid(h) if return_grid_data
+    fn = "lookup_grid_distinct"
+    hjson = "'#{h.to_json}'::JSONB"
+    rijson = "'#{row_info.to_json}'::JSONB"
+    params = "#{hjson}, #{rijson}, #{ret_grid_data}, #{distinct}"
+    sql = "SELECT #{fn}(#{params})"
+    raw = ActiveRecord::Base.connection.execute(sql)[0][fn]
+    res = JSON.parse(raw)
+
+    if res["error"]
+      msg = res["error"]
+      parms, sqls, ress, dg = res["error_extra"].values_at(
+                                "params", "sql",
+                                "results", "dg")
+      raise "DG #{name}: Error in PLV8 call: #{msg}\n"\
+            "params: #{parms}\n"\
+            "sqls: #{sqls}\n"\
+            "results: #{ress}\n"\
+            "dg: #{dg}\n"\
+            "ri: #{row_info}" if res["error"]
+    end
 
-    return {
-      "result"   => (data[vi][hi] if vi && hi),
-      "name"     => name,
-      "data"     => (modified_data if return_grid_data),
-      "metadata" => (modified_metadata if return_grid_data)
-    }
+    if ret_grid_data
+      md, mmd = modify_grid(h_passed)
+      res["data"] = md
+      res["metadata"] = mmd
+    end
+    res
   end
 
   # FIXME: added for Apollo -- not sure where this belongs given that
@@ -252,8 +201,7 @@ class Marty::DataGrid < Marty::Base
     raise "bad DataGrid #{dg}" unless Marty::DataGrid === dg
     raise "non-hash arg #{h}" unless Hash === h
 
-    res = dg.lookup_grid_distinct(pt, h, false, distinct)
-
+    res = dg.plv_lookup_grid_distinct(h, false, distinct)
     res["result"]
   end
 
@@ -283,7 +231,7 @@ class Marty::DataGrid < Marty::Base
     #   "data"     => <grid's data array>
     #   "metadata" => <grid's metadata (array of hashes)>
 
-    vhash = lookup_grid_distinct(pt, h, return_grid_data)
+    vhash = plv_lookup_grid_distinct(h, return_grid_data)
 
     return vhash if vhash["result"].nil? || !data_type
 
@@ -291,7 +239,11 @@ class Marty::DataGrid < Marty::Base
 
     return vhash if String === c_data_type
 
-    v = Marty::DataGrid.find_class_instance(pt, c_data_type, vhash["result"])
+    res = vhash["result"]
+
+    v =  Marty::PgEnum === res ?
+      c_data_type.find_by_name(res) :
+      Marty::DataConversion.find_row(c_data_type, {"name" => res}, pt)
 
     return vhash.merge({"result" => v}) unless (Marty::DataGrid === v && follow)
 

data/app/views/marty/diagnostic/op.html.erb

@@ -0,0 +1,58 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title><%=Rails.application.class.parent_name%> Diagnostic</title>
+    <style type="text/css">
+      body { margin: auto;
+             margin-bottom: 50px;
+             border: 0;
+             background-color: #fff;
+             color: #333333;
+             font-family: Arial, Helvetica, Verdana, sans-serif;
+             font-weight: normal;
+             font-size: 9pt; }
+      table { border: none;
+              border-collapse: collapse;
+              display: inline-block;
+              margin: 0px 5px 0px 5px;
+              text-align: left;
+      }
+      th { padding: 9px;
+           border: none;
+           background-color: #d6d6d6 }
+      td { padding: 9px;
+      border: none;
+      }
+      th.error  { background-color: #ff5555;
+                  color: #ffffff;
+      }
+      tr.passed { background-color: #d0e9c6 }
+      tr.failed { background-color: #ff5555;
+                  color: #ffffff
+      }
+      td.desc { font-size: 10pt; }
+      h1 { display: block;
+           margin: 0px auto 40px auto;
+           padding: 8px;
+           background-color: #1aaa55;
+           font-size: 18pt;
+           color: #ffffff;
+           line-height: 1.5em; }
+      h2 {text-align: center;}
+      h3.error {color: red;}
+      h3 {text-align: center;}
+      td.overflow { max-width: 350px;
+         overflow: auto; }
+      .wrapper {
+         overflow-x: auto;
+         white-space: nowrap;
+         display: block;}
+    </style>
+  </head>
+  <body>
+  <div style="text-align:center">
+    <h1><%=Rails.application.class.parent_name%> Diagnostic &#x26E8</h1>
+    <%== @result %>
+  </div>
+</body>
+</html>

data/config/routes.rb

@@ -2,6 +2,5 @@ Marty::Engine.routes.draw do
   match via: [:get, :post], "rpc/:action(.:format)" => "rpc", as: :rpc
   get "job/:action" => "job", as: :job
   match via: [:get, :post], "report(.:format)" => "report#index", as: :report
-  get  'diag/(:action)', controller: 'diagnostic'
-  get  'diagnostic/(:action)', controller: 'diagnostic'
+  get  'diag', to: 'diagnostic#op'
 end

data/db/js/errinfo_v1.js

@@ -0,0 +1,16 @@
+// PARAM: err JSONB
+// RETURN: JSONB
+var locre = /at.*[ (]([a-z][a-z0-9_]*[:][0-9]+)[:][0-9]+/i;
+var stack = err.stack;
+var res = '';
+if (stack) {
+    var lines = stack.split('\\n');
+    for (i=0, len=lines.length; i<len; ++i) {
+        m = locre.exec(lines[i]);
+        if (m) {
+            res += m[1];
+        }
+    }
+    return { "error": `${res} ${err.message}` };
+}
+else return { "error": err };

data/db/js/lookup_grid_distinct_v1.js

@@ -0,0 +1,64 @@
+// PARAM: h JSONB
+// PARAM: row_info JSONB
+// PARAM: return_grid_data boolean default false
+// PARAM: dis boolean default false
+// RETURN: JSONB
+var sqls = []
+var ress = []
+try {
+    var query_dir = plv8.find_function('query_grid_dir');
+    var errinfo = plv8.find_function('errinfo');
+    var ih = {};
+    var sql = 'SELECT metadata, lenient, name, group_id, data FROM marty_data_grids WHERE id = $1'
+    var dg = plv8.execute(sql, [row_info['id']])[0];
+    var res;
+    ['h','v'].forEach(function(dir) {
+        var infos = dg["metadata"].filter(function(md) { return md["dir"] == dir; });
+        if (infos.length == 0)
+        {
+            ih[dir] = [0]
+            return
+        }
+        a = query_dir(h, infos, row_info);
+        sqls.push(a);
+        ih[dir] = []
+        if (a) {
+            res = plv8.execute(a[0], a[1]);
+            ress.push(res);
+            for (var j = 0; j < res.length; j++)
+            {
+                ih[dir].push(res[j]["index"])
+            }
+        }
+        if (dis && ih[dir].length > 1)  {
+            throw Error("matches > 1");
+        }
+    });
+    if ((ih["v"].length == 0 || ih["h"].length == 0) &&
+        !dg['lenient'] && !return_grid_data) {
+        throw Error("Data Grid lookup failed");
+    }
+    
+    vi = ih["v"].length > 0 ? Math.min.apply(9999, ih["v"]) : null
+    hi = ih["h"].length > 0 ? Math.min.apply(9999, ih["h"]) : null
+    
+    var result = null;
+    if (vi!==null && hi!==null) {
+        result = dg["data"][vi][hi];
+    }
+    return { "result" : result,
+             "name"   : dg["name"],
+             "data"   : return_grid_data ? dg["data"] : null,
+             "metadata" : return_grid_data ? dg["metadata"] : null
+           };
+} catch (err) {
+    ei = errinfo(err);
+
+    ei["error_extra"] = {}
+    ei["error_extra"]["sql"] = sqls;
+    ei["error_extra"]["results"] = ress;
+    ei["error_extra"]["params"] = h;
+    ei["error_extra"]["dg"] = dg;
+
+    return ei;
+}

data/db/js/query_grid_dir_v1.js

@@ -0,0 +1,61 @@
+// PARAM: h JSONB
+// PARAM: infos JSONB[]
+// PARAM: row_info JSONB
+// RETURN: JSONB
+var getfilter = function(type, idx) {
+    switch(type) {
+    case "boolean":
+        return "key = $" + idx + " OR ";
+    case "numrange":
+        return "key @> $" + idx + "::numeric OR ";
+    case "int4range":
+        return "key @> $" + idx + "::integer OR ";
+    case "integer":
+        return "key @> ARRAY[$" + idx + "::integer] OR ";
+    default:
+        return "key @> ARRAY[$" + idx + "::text] OR ";
+    }
+}
+
+var temp = [];
+var args = [];
+
+var sql;
+for (var sqlidx=1, i = 0; i < infos.length; i++) {
+    var type = infos[i]["type"];
+    var attr = infos[i]["attr"];
+    var v = h[attr];
+    if (!h.hasOwnProperty(attr)) {
+        //throw Error(`missing attr ${attr}`)
+        continue;
+    }
+    switch (type) {
+    case 'boolean':
+    case "numrange":
+    case "int4range":
+    case "integer":
+        tab = `marty_grid_index_${type}s`;
+        break;
+    default:
+        tab = 'marty_grid_index_strings';
+    };
+
+    sql = `SELECT DISTINCT index from ${tab} ` +
+        "WHERE data_grid_id = $" + sqlidx++ +
+        " AND created_dt = $"    + sqlidx++ +
+        " AND attr = $"          + sqlidx++ + ' ';
+
+    args.push(row_info["group_id"]);
+    args.push(row_info["created_dt"]);
+    args.push(attr);
+
+    if (v!==null) {
+        filt = getfilter(type, sqlidx++);
+        args.push(v);
+    } else filt = ''
+    sql += ' AND (' + filt + "key is NULL) ";
+
+    temp.push(sql);
+}
+if (temp ==[]) return null;
+return [temp.join(" INTERSECT "), args];

data/db/migrate/400_create_dg_plv8_v1_fns.rb

@@ -0,0 +1,12 @@
+class CreateDgPlv8V1Fns < ActiveRecord::Migration
+  def change
+    connection.execute <<-SQL
+      -- required to utilize plv8 extension
+      CREATE EXTENSION IF NOT EXISTS plv8;
+    SQL
+    marty_path = Gem.loaded_specs["marty"].full_gem_path
+    Dir.glob("#{marty_path}/db/js/*_v1.js") do |f|
+      connection.execute(Marty::Migrations.get_plv8_migration(f))
+    end
+  end
+end

data/lib/marty/migrations.rb

@@ -303,4 +303,24 @@ OUT
                  "cols = #{cols}, act_cols = #{act_cols}")
     end.map(&:to_sym)
   end
+
+  def self.get_plv8_migration(file)
+    fnname = %r(/([^/]+)_v[0-9]+\.js\z).match(file)[1]
+    lines=File.readlines(file)
+    parts = lines.map do |line|
+      next [:param, $1] if %r(\A// PARAM[:] (.*)$).match(line)
+      next [:ret, $1]  if %r(\A// RETURN[:] (.*)$).match(line)
+      [:body, line]
+    end.group_by(&:first)
+    args = parts[:param].map{ |(_,l)| l}.join(",\n")
+    ret =  parts[:ret][0][1]
+    body = parts[:body].map{ |(_,l)| l}.join
+    <<EOT
+CREATE OR REPLACE FUNCTION #{fnname} (
+#{args}
+) RETURNS #{ret} AS $$
+#{body}
+$$ LANGUAGE plv8;
+EOT
+  end
 end