etna 0.1.32 → 0.1.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb19650be05ef97433ec00556981388436eb54ee48844055218f3ec290b10f3f
-  data.tar.gz: fbbee21325ddd316b4a742bc0cd143cba33c4bce635ad767720ed2d330da5446
+  metadata.gz: 484c950fffbafcf5132df923bc0ef88a79faa6c285ccde653805aac526e4a97e
+  data.tar.gz: f513b75b048e816acc494c09cb0eccbc2c38cda02c2723266fe7bd98c8595f3a
 SHA512:
-  metadata.gz: ee5548756441715e1944ced74613ae21f328910c3dcd982d638918f0df432d0ca8ee24b4ac62d7f86b17b3325483635e7e2b2ae750f93b82ef9a5629d925de01
-  data.tar.gz: ce866396a93210565bfc6bd9d5ff1acd69833c3bb9ff55a845e4fbdaad3769b65b0402ee3dc4c0ea937aacb7eac1432b9460ff5612c4344317bd4fcffaec4771
+  metadata.gz: 6ad0e99925dc6110c8f80f4a1e1417d610b28838497f09afe8be1c69a65fec95a7d549af987aaccde2b42480347bf336838be5398ef580d3024476206b4d611d
+  data.tar.gz: 44eb05596c635e7ab96df7971fed6b24bfee7bd545238e173e3d1b57bdcfe7b30db5deea355d2711b192d026d8fb72afc32cbcbcf4a83ef92848184433b7a813

data/etna.completion

@@ -32,7 +32,7 @@ arg_flag_completion_names="$arg_flag_completion_names  "
 multi_flags="$multi_flags  "
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
-all_completion_names="help models project"
+all_completion_names="help models project token"
 all_completion_names="$all_completion_names $all_flag_completion_names"
 if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
 return
@@ -722,6 +722,120 @@ else
 return
 fi
 done
+elif [[ "$1" == "token" ]]; then
+shift
+all_flag_completion_names="$all_flag_completion_names  "
+arg_flag_completion_names="$arg_flag_completion_names  "
+multi_flags="$multi_flags  "
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names="generate help"
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ "$1" == "generate" ]]; then
+shift
+all_flag_completion_names="$all_flag_completion_names --task --project-name "
+arg_flag_completion_names="$arg_flag_completion_names --project-name "
+multi_flags="$multi_flags  "
+declare _completions_for_project_name="__project_name__"
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names=""
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+if ! [[ "$multi_flags" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+fi
+a=$1
+shift
+if [[ "$arg_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
+return
+elif [[ "$1" == "help" ]]; then
+shift
+all_flag_completion_names="$all_flag_completion_names  "
+arg_flag_completion_names="$arg_flag_completion_names  "
+multi_flags="$multi_flags  "
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names=""
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+if ! [[ "$multi_flags" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+fi
+a=$1
+shift
+if [[ "$arg_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+if ! [[ "$multi_flags" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+fi
+a=$1
+shift
+if [[ "$arg_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
 elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
 return
 elif [[ "$all_flag_completion_names" =~ $1\  ]]; then

data/lib/commands.rb

@@ -91,6 +91,36 @@ class EtnaApp
   class Administrate
     include Etna::CommandExecutor
 
+    class Token
+      include Etna::CommandExecutor
+
+      class Generate < Etna::Command
+        include WithLogger
+
+        boolean_flags << "--task"
+        string_flags << "--project-name"
+        string_flags << "--email"
+
+        def execute(email:, task: false, project_name: nil)
+          # the token is not required, but can be used if available
+          # to generate a task token, so we pass it in here
+          janus_client = Etna::Clients::Janus.new(
+            token: ENV['TOKEN'],
+            ignore_ssl: EtnaApp.instance.config(:ignore_ssl),
+            **EtnaApp.instance.config(:janus, EtnaApp.instance.environment))
+
+          generate_token_workflow = Etna::Clients::Janus::GenerateTokenWorkflow.new(
+            janus_client: janus_client,
+            token_type: task ? 'task' : 'login',
+            email: email,
+            project_name: project_name,
+            private_key_file: EtnaApp.instance.config(:private_key, EtnaApp.instance.environment)
+          )
+          generate_token_workflow.generate!
+        end
+      end
+    end
+
     class Project
       include Etna::CommandExecutor
 
@@ -309,7 +339,7 @@ class EtnaApp
             request = Etna::Clients::Magma::RetrievalRequest.new(project_name: project_name)
             request.model_name = model_name
             request.attribute_names = 'all'
-            request.record_names = 'all'
+            request.record_names = []
             model = magma_client.retrieve(request).models.model(model_name)
             model_parent_name = model.template.attributes.all.select do |attribute|
               attribute.attribute_type == Etna::Clients::Magma::AttributeType::PARENT

data/lib/etna.rb

@@ -21,6 +21,7 @@ require_relative './etna/environment_scoped'
 require_relative './etna/filesystem'
 require_relative './etna/formatting'
 require_relative './etna/cwl'
+require_relative './etna/metrics'
 
 class EtnaApp
   include Etna::Application

data/lib/etna/application.rb

@@ -8,6 +8,7 @@ require_relative './command'
 require_relative './generate_autocompletion_script'
 require 'singleton'
 require 'rollbar'
+require 'fileutils'
 
 module Etna::Application
   def self.included(other)
@@ -31,6 +32,12 @@ module Etna::Application
     raise "Could not find application instance from #{namespace}, and not subclass of Application found."
   end
 
+  # Used to find the application in development recorded vcr tests.
+  # see spec/vcr.rb
+  def dev_route
+    "#{self.class.name.split('::').first.downcase}.development.local"
+  end
+
   def self.register(app)
     @instance = app
   end
@@ -53,6 +60,23 @@ module Etna::Application
     end
   end
 
+  def setup_yabeda
+    Yabeda.configure!
+  end
+
+  def write_job_metrics(name)
+    node_metrics_dir = config(:node_metrics_dir) || "/tmp/metrics.prom"
+    ::FileUtils.mkdir_p(node_metrics_dir)
+
+    tmp_file = ::File.join(node_metrics_dir, "#{name}.prom.$$")
+    ::File.open(tmp_file, "w") do |f|
+      f.write(Prometheus::Client::Formats::Text.marshal(Prometheus::Client.registry))
+    end
+
+    require 'fileutils'
+    ::FileUtils.mv(tmp_file, ::File.join(node_metrics_dir, "#{name}.prom"))
+  end
+
   def setup_logger
     @logger = Etna::Logger.new(
       # The name of the log_file, required.

data/lib/etna/auth.rb

@@ -70,6 +70,29 @@ module Etna
       return route && route.noauth?
     end
 
+    def janus_approved?(payload, token, request)
+      route = server.find_route(request)
+
+      # some routes don't need janus approval
+      return true if route && route.ignore_janus?
+
+      # only process task tokens right now
+      return true unless payload['task']
+
+      return false unless application.config(:janus) && application.config(:janus)[:host]
+
+      janus_client = Etna::Clients::Janus.new(
+        token: token,
+        host: application.config(:janus)[:host]
+      )
+
+      response = janus_client.validate_task_token
+
+      return false unless response.code == '200'
+
+      return true
+    end
+
     def approve_user(request)
       token = request.cookies[application.config(:token_name)] || auth(request, :etna)
 
@@ -77,6 +100,8 @@ module Etna
 
       begin
         payload, header = application.sign.jwt_decode(token)
+
+        return false unless janus_approved?(payload, token, request)
         return request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
       rescue
         # bail out if anything goes wrong

data/lib/etna/client.rb

@@ -17,6 +17,13 @@ module Etna
 
     attr_reader :routes
 
+    def with_headers(headers, &block)
+      @request_headers = headers.compact
+      result = instance_eval(&block)
+      @request_headers = nil
+      return result
+    end
+
     def signed_route_path(route, params)
       path = route_path(route,params)
 
@@ -111,7 +118,7 @@ module Etna
 
     def body_request(type, endpoint, params = {}, &block)
       uri = request_uri(endpoint)
-      req = type.new(uri.request_uri, request_params)
+      req = type.new(uri.request_uri, request_headers)
       req.body = params.to_json
       request(uri, req, &block)
     end
@@ -124,7 +131,7 @@ module Etna
       else
         uri.query = URI.encode_www_form(params)
       end
-      req = type.new(uri.request_uri, request_params)
+      req = type.new(uri.request_uri, request_headers)
       request(uri, req, &block)
     end
 
@@ -132,12 +139,14 @@ module Etna
       URI("#{@host}#{endpoint}")
     end
 
-    def request_params
+    def request_headers
       {
           'Content-Type' => 'application/json',
           'Accept' => 'application/json, text/*',
           'Authorization' => "Etna #{@token}"
-      }
+      }.update(
+        @request_headers || {}
+      )
     end
 
     def status_check!(response)
@@ -164,7 +173,7 @@ module Etna
         verify_mode = @ignore_ssl ?
           OpenSSL::SSL::VERIFY_NONE :
           OpenSSL::SSL::VERIFY_PEER
-        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
+        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
           http.request(data) do |response|
             status_check!(response)
             yield response
@@ -174,7 +183,7 @@ module Etna
         verify_mode = @ignore_ssl ?
           OpenSSL::SSL::VERIFY_NONE :
           OpenSSL::SSL::VERIFY_PEER
-        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
+        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
           response = http.request(data)
           status_check!(response)
           return response

data/lib/etna/clients/base_client.rb

@@ -8,10 +8,9 @@ module Etna
       attr_reader :host, :token, :ignore_ssl
       def initialize(host:, token:, ignore_ssl: false)
         raise "#{self.class.name} client configuration is missing host." unless host
-        raise "#{self.class.name} client configuration is missing token." unless token
 
         @token = token
-        raise "Your token is expired." if token_expired?
+        raise "Your token is expired." if token && token_expired?
 
         @etna_client = ::Etna::Client.new(
           host,
@@ -30,10 +29,11 @@ module Etna
       def token_will_expire?(offset=3000)
         # offset in seconds
         # Will the user's token expire in the given amount of time?
-        epoch_seconds = JSON.parse(Base64.urlsafe_decode64(token.split('.')[1]))["exp"]
+        payload = JSON.parse(Base64.urlsafe_decode64(token.split('.')[1]))
+        epoch_seconds = payload["exp"]
         expiration = DateTime.strptime(epoch_seconds.to_s, "%s")
         expiration <= DateTime.now.new_offset + offset
       end
     end
   end
-end
+end

data/lib/etna/clients/janus.rb

@@ -1,2 +1,3 @@
 require_relative './janus/client'
 require_relative './janus/models'
+require_relative './janus/workflows'

data/lib/etna/clients/janus/client.rb

@@ -57,6 +57,24 @@ module Etna
 
         TokenResponse.new(token)
       end
+
+      def validate_task_token
+        @etna_client.post('/api/tokens/validate_task')
+      end
+
+      def get_nonce
+        @etna_client.get('/api/tokens/nonce').body
+      end
+
+      def generate_token(token_type, signed_nonce: nil, project_name: nil, read_only: false)
+        response = @etna_client.with_headers(
+          'Authorization' => signed_nonce ? "Signed-Nonce #{signed_nonce}" : nil
+        ) do
+          post('/api/tokens/generate', token_type: token_type, project_name: project_name, read_only: read_only)
+        end
+
+        response.body
+      end
     end
   end
 end

data/lib/etna/clients/janus/models.rb

@@ -51,6 +51,12 @@ module Etna
         end
       end
 
+      class ValidateTaskTokenRequest
+        def map
+          []
+        end
+      end
+
       class HtmlResponse
         attr_reader :raw
 
@@ -76,4 +82,4 @@ module Etna
       end
     end
   end
-end
+end

data/lib/etna/clients/janus/workflows.rb

@@ -0,0 +1 @@
+require_relative './workflows/generate_token_workflow'

data/lib/etna/clients/janus/workflows/generate_token_workflow.rb

@@ -0,0 +1,77 @@
+# Base workflow for setting up a project by a super user.
+# 1) Creates the project in janus
+# 2) Adds administrator(s) to Janus
+# 3) Refreshes the user's token with the new privileges.
+# 4) Creates the project in .
+
+require 'base64'
+require 'json'
+require 'ostruct'
+require_relative '../models'
+
+module Etna
+  module Clients
+    class Janus
+      class GenerateTokenWorkflow < Struct.new(:janus_client, :email, :project_name, :token_type, :private_key_file, keyword_init: true)
+        def generate!
+          nonce = janus_client.get_nonce
+
+          unless email
+            puts "Email address for #{janus_client.host} account?"
+            email = STDIN.gets.chomp
+          end
+
+          if use_nonce?
+            until private_key_file
+              puts "Location of private key file?"
+              private_key_file = ::File.expand_path(STDIN.gets.chomp)
+              unless File.exists?(private_key_file)
+                puts "No such file."
+                private_key_file = nil
+              end
+            end
+          end
+
+          if needs_project_name?
+            puts "Project name?"
+            project_name = STDIN.gets.chomp
+          end
+          
+          token = janus_client.generate_token(token_type, signed_nonce: use_nonce? ? signed_nonce(nonce) : nil, project_name: project_name)
+
+          puts token
+        end
+
+        private
+
+        def signed_nonce(nonce)
+          private_key = OpenSSL::PKey::RSA.new(File.read(private_key_file))
+
+          txt_to_sign = "#{nonce}.#{Base64.strict_encode64(email)}"
+
+          sig = Base64.strict_encode64(
+            private_key.sign(OpenSSL::Digest::SHA256.new,txt_to_sign)
+          )
+
+          "#{txt_to_sign}.#{sig}"
+        end
+
+        def use_nonce?
+          !task_token? || !janus_client.token
+        end
+
+        def needs_project_name?
+          task_token? && !project_name
+        end
+
+        def task_token?
+          token_type == 'task'
+        end
+
+        def user
+          @user ||= JSON.parse(Base64.urlsafe_decode64(magma_client.token.split('.')[1]))
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/magma/formatting/models_csv.rb

@@ -138,7 +138,7 @@ module Etna
                 attribute_type: attribute.attribute_type,
                 link_model_name: attribute.link_model_name,
                 reciprocal_link_type: models.find_reciprocal(model: model, attribute: attribute)&.attribute_type,
-                description: attribute.desc,
+                description: attribute.description,
                 display_name: attribute.display_name,
                 match: attribute.match,
                 format_hint: attribute.format_hint,
@@ -175,7 +175,7 @@ module Etna
           # This should line up with the attribute names _on the model itself_.
           ATTRIBUTE_ROW_ENTRIES = [
               :attribute_type,
-              :link_model_name, :desc,
+              :link_model_name, :description,
               :display_name, :format_hint,
               :restricted, :read_only,
               :validation, :attribute_group,
@@ -188,7 +188,6 @@ module Etna
 
           def format_row(row)
             replace_row_column(row, :attribute_type) { |s| AttributeType.new(s) }
-            replace_row_column(row, :desc) { row.delete(:description) }
             replace_row_column(row, :restricted, &COLUMN_AS_BOOLEAN)
             replace_row_column(row, :read_only, &COLUMN_AS_BOOLEAN)
             replace_row_column(row, :options) { |s| {"type" => "Array", "value" => s.split(',').map(&:strip)} }
@@ -250,7 +249,7 @@ module Etna
                 parent_att.name = parent_att.attribute_name = parent_model_name
                 parent_att.attribute_type = Etna::Clients::Magma::AttributeType::PARENT
                 parent_att.link_model_name = parent_model_name
-                parent_att.desc = prettify(parent_model_name)
+                parent_att.description = prettify(parent_model_name)
                 parent_att.display_name = prettify(parent_model_name)
               end
             end
@@ -275,7 +274,7 @@ module Etna
                   attr.attribute_name = attr.name = template.name
                   attr.attribute_type = parent_link_type
                   attr.link_model_name = template.name
-                  attr.desc = prettify(template.name)
+                  attr.description = prettify(template.name)
                   attr.display_name = prettify(template.name)
                 end
               end
@@ -338,7 +337,7 @@ module Etna
                     models.build_model(att.link_model_name).build_template.build_attributes.build_attribute(template.name).tap do |rec_att|
                       rec_att.attribute_name = rec_att.name = template.name
                       rec_att.display_name = prettify(template.name)
-                      rec_att.desc = prettify(template.name)
+                      rec_att.description = prettify(template.name)
                       rec_att.attribute_type = Etna::Clients::Magma::AttributeType::COLLECTION
                       rec_att.link_model_name = template.name
                     end

data/lib/etna/clients/magma/formatting/models_odm_xml.rb

@@ -246,7 +246,7 @@ module Etna
               }
 
               params['redcap:TextValidationType'] = redcap_text_validation_map[attribute_type] if redcap_text_validation_map[attribute_type]
-              params['redcap:FieldNote'] = attribute.desc if attribute.desc
+              params['redcap:FieldNote'] = attribute.description if attribute.description
               xml.ItemDef(params) do
                 xml.Question do
                   xml.send('TranslatedText', attribute_name.capitalize)

data/lib/etna/clients/magma/models.rb

@@ -10,7 +10,7 @@ require_relative '../base_client'
 module Etna
   module Clients
     class Magma < Etna::Clients::BaseClient
-      class RetrievalRequest < Struct.new(:model_name, :attribute_names, :record_names, :project_name, :page, :page_size, :order, :filter, keyword_init: true)
+      class RetrievalRequest < Struct.new(:model_name, :attribute_names, :record_names, :project_name, :page, :page_size, :order, :filter, :hide_templates, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**params)
@@ -18,7 +18,7 @@ module Etna
         end
       end
 
-      class QueryRequest < Struct.new(:query, :project_name, keyword_init: true)
+      class QueryRequest < Struct.new(:query, :project_name, :order, :page, :page_size, keyword_init: true)
         include JsonSerializableStruct
       end
 
@@ -117,14 +117,6 @@ module Etna
           super({action_name: 'update_attribute'}.update(args))
         end
 
-        def desc=(val)
-          self.description = val
-        end
-
-        def desc
-          self.description
-        end
-
         def as_json
           super(keep_nils: true)
         end
@@ -454,6 +446,16 @@ module Etna
           @raw = raw
         end
 
+        def is_edited?(other)
+          # Don't just override == in case need to do a full comparison.
+          editable_attribute_names = Attribute::EDITABLE_ATTRIBUTE_ATTRIBUTES.map(&:to_s)
+          
+          self_editable = raw.slice(*editable_attribute_names)
+          other_editable = other.raw.slice(*editable_attribute_names)
+
+          self_editable != other_editable
+        end
+
         # Sets certain attribute fields which are implicit, even when not set, to match server behavior.
         def set_field_defaults!
           @raw.replace({
@@ -511,24 +513,12 @@ module Etna
           raw['unique'] = val
         end
 
-        def desc
-          raw['desc']
-        end
-
-        def desc=(val)
-          @raw['desc'] = val
-        end
-
-        # description and description= are needed
-        #   to make UpdateAttribute actions
-        #   work in the model_synchronization_workflow for
-        #   desc.
         def description
-          raw['desc']
+          raw['description']
         end
 
         def description=(val)
-          @raw['desc'] = val
+          @raw['description'] = val
         end
 
         def display_name
@@ -595,11 +585,8 @@ module Etna
           raw['options']
         end
 
-        # NOTE!  The Attribute class returns description as desc, where as actions take it in as description.
-        # There are shortcut methods that try to handle this on the action class side of things.  Ideally we would
-        # make this more consistent in the near future.
         COPYABLE_ATTRIBUTE_ATTRIBUTES = [
-            :attribute_name, :attribute_type, :desc, :display_name, :format_hint,
+            :attribute_name, :attribute_type, :display_name, :format_hint,
             :hidden, :link_model_name, :read_only, :attribute_group, :unique, :validation,
             :restricted, :description
         ]

data/lib/etna/clients/magma/workflows/file_linking_workflow.rb

@@ -95,7 +95,9 @@ module Etna
             file_path = ::File.dirname(file_path)
             {attribute_name => "https://metis.ucsf.edu/#{project_name}/browse/#{bucket_name}/#{file_path}"}
           else
-            {attribute_name => {path: "metis://#{project_name}/#{bucket_name}/#{file_path}"}}
+            {attribute_name => {
+              path: "metis://#{project_name}/#{bucket_name}/#{file_path}",
+              original_filename: File.basename(file_path)}}
           end
         end
 

data/lib/etna/clients/magma/workflows/model_synchronization_workflow.rb

@@ -190,10 +190,7 @@ module Etna
             source_attribute = Attribute.new(source_attribute.raw)
             source_attribute.set_field_defaults!
 
-            source_editable = source_attribute.raw.slice(*Attribute::EDITABLE_ATTRIBUTE_ATTRIBUTES.map(&:to_s))
-            target_editable = target_attribute.raw.slice(*Attribute::EDITABLE_ATTRIBUTE_ATTRIBUTES.map(&:to_s))
-
-            if source_editable == target_editable
+            if !source_attribute.is_edited?(target_attribute)
               return
             end
 

data/lib/etna/clients/magma/workflows/walk_model_tree_workflow.rb

@@ -12,14 +12,37 @@ module Etna
           @template_for = {}
         end
 
+        def all_attributes(template:)
+          template.attributes.attribute_keys
+        end
+
+        def safe_group_attributes(template:, attributes_mask:)
+          result = []
+
+          template.attributes.attribute_keys.each do |attribute_name|
+            next if template.attributes.attribute(attribute_name).attribute_type == Etna::Clients::Magma::AttributeType::TABLE
+            result << attribute_name if attribute_included?(attributes_mask, attribute_name)
+          end
+
+          result
+        end
+
         def masked_attributes(template:, model_attributes_mask:, model_name:)
           attributes_mask = model_attributes_mask[model_name]
-          return ["all", "all"] if attributes_mask.nil?
-          [(attributes_mask + [template.identifier, 'parent']).uniq, attributes_mask]
+
+          if attributes_mask.nil?
+            return [
+              safe_group_attributes(template: template, attributes_mask: attributes_mask),
+              all_attributes(template: template)
+            ]
+          end
+
+          [(safe_group_attributes(template: template, attributes_mask: attributes_mask) + [template.identifier, 'parent']).uniq,
+            attributes_mask]
         end
 
         def attribute_included?(mask, attribute_name)
-          return true if mask == "all"
+          return true if mask.nil?
           mask.include?(attribute_name)
         end
 
@@ -48,7 +71,11 @@ module Etna
             seen.add([path[:from], model_name])
 
             template = template_for(model_name)
-            query_attributes, walk_attributes = masked_attributes(template: template, model_attributes_mask: model_attributes_mask, model_name: model_name)
+            query_attributes, walk_attributes = masked_attributes(
+              template: template,
+              model_attributes_mask: model_attributes_mask,
+              model_name: model_name
+            )
 
             request = RetrievalRequest.new(
                 project_name: magma_crud.project_name,
@@ -62,6 +89,7 @@ module Etna
             related_models = {}
 
             magma_crud.page_records(model_name, request) do |response|
+              logger&.info("Fetched page of #{model_name}")
               tables = []
               collections = []
               links = []
@@ -70,13 +98,15 @@ module Etna
               model = response.models.model(model_name)
 
               template.attributes.attribute_keys.each do |attr_name|
+                attr = template.attributes.attribute(attr_name)
+                if attr.attribute_type == AttributeType::TABLE && attribute_included?(walk_attributes, attr_name)
+                  tables << attr_name
+                end
+
                 next unless attribute_included?(query_attributes, attr_name)
                 attributes << attr_name
 
-                attr = template.attributes.attribute(attr_name)
-                if attr.attribute_type == AttributeType::TABLE
-                  tables << attr_name
-                elsif attr.attribute_type == AttributeType::COLLECTION
+                if attr.attribute_type == AttributeType::COLLECTION
                   related_models[attr.link_model_name] ||= Set.new
                   collections << attr_name
                 elsif attr.attribute_type == AttributeType::LINK
@@ -91,14 +121,32 @@ module Etna
                 end
               end
 
+              table_data = {}
+              # Request tables in an inner chunk.
+              tables.each do |table_attr|
+                request = RetrievalRequest.new(
+                  project_name: magma_crud.project_name,
+                  model_name: model_name,
+                  record_names: model.documents.document_keys,
+                  attribute_names: [table_attr],
+                )
+
+                logger&.info("Fetching inner table #{table_attr}...")
+
+                table_response = magma_crud.magma_client.retrieve(request)
+                d = table_response.models.model(model_name).documents
+                table_d = table_response.models.model(table_attr).documents
+                table_data[table_attr] = d.document_keys.map do |id|
+                  [id, d.document(id)[table_attr].map { |tid| table_d.document(tid) }]
+                end.to_h
+              end
+
               model.documents.document_keys.each do |key|
                 record = model.documents.document(key).slice(*attributes)
 
                 # Inline tables inside the record
                 tables.each do |table_attr|
-                  record[table_attr] = record[table_attr].map do |id|
-                    response.models.model(template.attributes.attribute(table_attr).link_model_name).documents.document(id)
-                  end unless record[table_attr].nil?
+                  record[table_attr] = table_data[table_attr][key] unless table_data[table_attr].nil?
                 end
 
                 collections.each do |collection_attr|

data/lib/etna/command.rb

@@ -203,6 +203,7 @@ module Etna
       @subcommands ||= self.class.constants.sort.reduce({}) do |acc, n|
         acc.tap do
           c = self.class.const_get(n)
+          next unless c.respond_to?(:instance_methods)
           next unless c.instance_methods.include?(:find_command)
           v = c.new(self)
           acc[v.command_name] = v

data/lib/etna/directed_graph.rb

@@ -7,6 +7,64 @@ class DirectedGraph
   attr_reader :children
   attr_reader :parents
 
+  def full_parentage(n)
+    [].tap do |result|
+      q = @parents[n].keys.dup
+      seen = Set.new
+
+      until q.empty?
+        n = q.shift
+        next if seen.include?(n)
+        seen.add(n)
+
+        result << n
+        q.push(*@parents[n].keys)
+      end
+
+      result.uniq!
+    end
+  end
+
+  def as_normalized_hash(root, include_root = true)
+    q = [root]
+    {}.tap do |result|
+      if include_root
+        result[root] = []
+      end
+
+      seen = Set.new
+
+      until q.empty?
+        n = q.shift
+        next if seen.include?(n)
+        seen.add(n)
+
+        parentage = full_parentage(n)
+
+        @children[n].keys.each do |child_node|
+          q << child_node
+
+          if result.include?(n)
+            result[n] << child_node
+          end
+
+          parentage.each do |grandparent|
+            result[grandparent] << child_node if result.include?(grandparent)
+          end
+
+          # Depending on the graph shape, diamonds could lead to
+          #   resetting of previously calculated dependencies.
+          # Here we avoid resetting existing entries in `result`
+          #   and instead concatenate them if they already exist.
+          result[child_node] = [] unless result.include?(child_node)
+          result[n].concat(result[child_node]) if result.include?(child_node) && result.include?(n)
+        end
+      end
+
+      result.values.each(&:uniq!)
+    end
+  end
+
   def add_connection(parent, child)
     children = @children[parent] ||= {}
     child_children = @children[child] ||= {}
@@ -18,12 +76,13 @@ class DirectedGraph
     parents[parent] = parent_parents
   end
 
-  def serialized_path_from(root)
+  def serialized_path_from(root, include_root = true)
     seen = Set.new
     [].tap do |result|
-      result << root
+      result << root if include_root
       seen.add(root)
-      path_q = paths_from(root)
+      path_q = paths_from(root, include_root)
+      traversables = path_q.flatten
 
       until path_q.empty?
         next_path = path_q.shift
@@ -34,7 +93,7 @@ class DirectedGraph
           next if next_n.nil?
           next if seen.include?(next_n)
 
-          if @parents[next_n].keys.any? { |p| !seen.include?(p) }
+          if @parents[next_n].keys.any? { |p| !seen.include?(p) && traversables.include?(p) }
             next_path.unshift(next_n)
             path_q.push(next_path)
             break
@@ -47,9 +106,9 @@ class DirectedGraph
     end
   end
 
-  def paths_from(root)
+  def paths_from(root, include_root = true)
     [].tap do |result|
-      parents_of_map = descendants(root)
+      parents_of_map = descendants(root, include_root)
       seen = Set.new
 
       parents_of_map.to_a.sort_by { |k, parents| [-parents.length, k.inspect] }.each do |k, parents|
@@ -68,7 +127,7 @@ class DirectedGraph
     end
   end
 
-  def descendants(parent)
+  def descendants(parent, include_root = true)
     seen = Set.new
 
     seen.add(parent)
@@ -90,7 +149,7 @@ class DirectedGraph
     while child = queue.pop
       next if seen.include? child
       seen.add(child)
-      path = (paths[child] ||= [parent])
+      path = (paths[child] ||= (include_root ? [parent] : []))
 
       @children[child].keys.each do |child_child|
         queue.push child_child

data/lib/etna/metrics.rb

@@ -0,0 +1,26 @@
+
+module Etna
+  class MetricsExporter
+    def initialize(app, path: '/metrics')
+      @app = app
+      @path = path
+    end
+
+    def exporter
+      @exporter ||= begin
+        exporter = Yabeda::Prometheus::Exporter.new(@app, path: @path)
+        Rack::Auth::Basic.new(exporter) do |user, pw|
+          user == 'prometheus' && pw == ENV['METRICS_PW']
+        end
+      end
+    end
+
+    def call(env)
+      if env['PATH_INFO'] == @path
+        exporter.call(env)
+      else
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/etna/route.rb

@@ -91,6 +91,10 @@ module Etna
       @auth && @auth[:noauth]
     end
 
+    def ignore_janus?
+      @auth && @auth[:ignore_janus]
+    end
+
     private
 
     def application
@@ -144,7 +148,10 @@ module Etna
       params = request.env['rack.request.params']
 
       @auth[:user].all? do |constraint, param_name|
-        user.respond_to?(constraint) && user.send(constraint, params[param_name])
+        user.respond_to?(constraint) && (
+          param_name.is_a?(Symbol) ?
+            user.send(constraint, params[param_name]) :
+            user.send(constraint, param_name))
       end
     end
 

data/lib/etna/server.rb

@@ -74,6 +74,11 @@ module Etna
     def initialize
       # Setup logging.
       application.setup_logger
+
+      # This needs to be required before yabeda invocation, but cannot belong at the top of any module since clients
+      # do not install yabeda.
+      require 'yabeda'
+      application.setup_yabeda
     end
 
     private

data/lib/etna/spec/vcr.rb

@@ -4,13 +4,32 @@ require 'openssl'
 require 'digest/sha2'
 require 'base64'
 
-def setup_base_vcr(spec_helper_dir)
+def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
   VCR.configure do |c|
     c.hook_into :webmock
     c.cassette_serializers
     c.cassette_library_dir = ::File.join(spec_helper_dir, 'fixtures', 'cassettes')
     c.allow_http_connections_when_no_cassette = true
 
+    c.register_request_matcher :verify_uri_route do |request_1, request_2|
+      next true if server.nil? || application.nil?
+
+      route_match = request_1.uri =~ /https:\/\/#{application.dev_route}(.*)/
+      if route_match && route_match[1]
+        def request_1.request_method
+          method.to_s.upcase
+        end
+
+        def request_1.path
+          URI(uri).path
+        end
+
+        !!server.find_route(request_1)
+      else
+        true
+      end
+    end
+
     c.register_request_matcher :try_body do |request_1, request_2|
       if request_1.headers['Content-Type'].first =~ /application\/json/
         if request_2.headers['Content-Type'].first =~ /application\/json/
@@ -39,7 +58,7 @@ def setup_base_vcr(spec_helper_dir)
         else
           ENV['RERECORD'] ? :all : :once
         end,
-        match_requests_on: [:method, :uri, :try_body]
+        match_requests_on: [:method, :uri, :try_body, :verify_uri_route]
     }
 
     # Filter the authorization headers of any request by replacing any occurrence of that request's

data/lib/etna/test_auth.rb

@@ -42,7 +42,6 @@ module Etna
       # We do this to support Metis client tests, we pass in tokens with multiple "."-separated parts, so
       #   have to account for that.
       payload = JSON.parse(Base64.decode64(token.split('.')[1]))
-
       request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
     end
 

data/lib/etna/user.rb

@@ -7,7 +7,7 @@ module Etna
     }
 
     def initialize params, token=nil
-      @name, @email, @encoded_permissions, encoded_flags = params.values_at(:name, :email, :perm, :flags)
+      @name, @email, @encoded_permissions, encoded_flags, @task = params.values_at(:name, :email, :perm, :flags, :task)
 
       @flags = encoded_flags&.split(/;/) || []
       @token = token unless !token
@@ -16,6 +16,10 @@ module Etna
 
     attr_reader :name, :email, :token
 
+    def task?
+      !!@task
+    end
+
     def permissions
       @permissions ||= @encoded_permissions.split(/\;/).map do |roles|
         role, projects = roles.split(/:/)

data/lib/helpers.rb

@@ -57,9 +57,9 @@ module WithEtnaClients
         **EtnaApp.instance.config(:metis, environment) || {})
   end
 
-  def janus_client
+  def janus_client(opts={})
     @janus_client ||= Etna::Clients::Janus.new(
-        token: token,
+        token: opts.has_key?(:token) ? opts[:token] : token,
         ignore_ssl: EtnaApp.instance.config(:ignore_ssl),
         **EtnaApp.instance.config(:janus, environment) || {})
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.32
+  version: 0.1.37
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-05 00:00:00.000000000 Z
+date: 2021-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -94,20 +94,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: concurrent-ruby-ext
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: See summary
 email: Saurabh.Asthana@ucsf.edu
 executables:
@@ -131,6 +117,8 @@ files:
 - lib/etna/clients/janus.rb
 - lib/etna/clients/janus/client.rb
 - lib/etna/clients/janus/models.rb
+- lib/etna/clients/janus/workflows.rb
+- lib/etna/clients/janus/workflows/generate_token_workflow.rb
 - lib/etna/clients/magma.rb
 - lib/etna/clients/magma/client.rb
 - lib/etna/clients/magma/formatting.rb
@@ -180,6 +168,7 @@ files:
 - lib/etna/hmac.rb
 - lib/etna/json_serializable_struct.rb
 - lib/etna/logger.rb
+- lib/etna/metrics.rb
 - lib/etna/multipart_serializable_nested_hash.rb
 - lib/etna/parse_body.rb
 - lib/etna/route.rb
@@ -213,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 2.7.6.3
 signing_key: 
 specification_version: 4
 summary: Base classes for Mount Etna applications