etna 0.1.30 → 0.1.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9118a99396cf2b041083ff982fb680a5d60096802699789962d19bcf3fecf41d
-  data.tar.gz: 3145994427f0fa57bf796e68dfe4e69156eb407f144a6afe048bd17454201949
+  metadata.gz: bf1f86e15c74113a4591afd3d8a43064b6ed3b448bdfad3739baa27061df7913
+  data.tar.gz: ed543c8775c89679b4755ca3119899639fc125df2d7e65c3515998f06ae45de8
 SHA512:
-  metadata.gz: ec59fd21bcf8bb88848e0c8c628a0185226d7dd7ee1af79525a790ac2117fe2b7d44b9d3e514717e9c23233bd531c9d3c8404981892174adbeee221e62118c33
-  data.tar.gz: bf0cc8cea804bb218e7f487ea014c84e394738760dc400de1c716b780da6a668ca5f853a72acfff7a77cd979f4a90004a866cdd16cb26a008a3498c83161fd71
+  metadata.gz: 8523211434696c74c03ca5319a089c74dba9bc3c3cc086f723dcadd1789c91f2e6c7fe43701cd3ac467460bc5a8ee1f4c82c032a5d0ac16e666a565212fba209
+  data.tar.gz: 60e3338c3021ea2e1a54dcfb7f2a15c9ffce71aed5a12db7739992649cfd9518f19edc49262a0095ba09c8783e0ef499bcb3015aebca0d2b7751e7096cbf88eb

data/etna.completion

@@ -32,7 +32,7 @@ arg_flag_completion_names="$arg_flag_completion_names  "
 multi_flags="$multi_flags  "
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
-all_completion_names="help models project"
+all_completion_names="help models project token"
 all_completion_names="$all_completion_names $all_flag_completion_names"
 if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
 return
@@ -722,6 +722,120 @@ else
 return
 fi
 done
+elif [[ "$1" == "token" ]]; then
+shift
+all_flag_completion_names="$all_flag_completion_names  "
+arg_flag_completion_names="$arg_flag_completion_names  "
+multi_flags="$multi_flags  "
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names="generate help"
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ "$1" == "generate" ]]; then
+shift
+all_flag_completion_names="$all_flag_completion_names --task --project-name "
+arg_flag_completion_names="$arg_flag_completion_names --project-name "
+multi_flags="$multi_flags  "
+declare _completions_for_project_name="__project_name__"
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names=""
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+if ! [[ "$multi_flags" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+fi
+a=$1
+shift
+if [[ "$arg_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
+return
+elif [[ "$1" == "help" ]]; then
+shift
+all_flag_completion_names="$all_flag_completion_names  "
+arg_flag_completion_names="$arg_flag_completion_names  "
+multi_flags="$multi_flags  "
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names=""
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+if ! [[ "$multi_flags" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+fi
+a=$1
+shift
+if [[ "$arg_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+if ! [[ "$multi_flags" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+fi
+a=$1
+shift
+if [[ "$arg_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
 elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
 return
 elif [[ "$all_flag_completion_names" =~ $1\  ]]; then

data/lib/commands.rb

@@ -91,6 +91,36 @@ class EtnaApp
   class Administrate
     include Etna::CommandExecutor
 
+    class Token
+      include Etna::CommandExecutor
+
+      class Generate < Etna::Command
+        include WithLogger
+
+        boolean_flags << "--task"
+        string_flags << "--project-name"
+        string_flags << "--email"
+
+        def execute(email:, task: false, project_name: nil)
+          # the token is not required, but can be used if available
+          # to generate a task token, so we pass it in here
+          janus_client = Etna::Clients::Janus.new(
+            token: ENV['TOKEN'],
+            ignore_ssl: EtnaApp.instance.config(:ignore_ssl),
+            **EtnaApp.instance.config(:janus, EtnaApp.instance.environment))
+
+          generate_token_workflow = Etna::Clients::Janus::GenerateTokenWorkflow.new(
+            janus_client: janus_client,
+            token_type: task ? 'task' : 'login',
+            email: email,
+            project_name: project_name,
+            private_key_file: EtnaApp.instance.config(:private_key, EtnaApp.instance.environment)
+          )
+          generate_token_workflow.generate!
+        end
+      end
+    end
+
     class Project
       include Etna::CommandExecutor
 
@@ -309,7 +339,7 @@ class EtnaApp
             request = Etna::Clients::Magma::RetrievalRequest.new(project_name: project_name)
             request.model_name = model_name
             request.attribute_names = 'all'
-            request.record_names = 'all'
+            request.record_names = []
             model = magma_client.retrieve(request).models.model(model_name)
             model_parent_name = model.template.attributes.all.select do |attribute|
               attribute.attribute_type == Etna::Clients::Magma::AttributeType::PARENT

data/lib/etna/auth.rb

@@ -70,6 +70,29 @@ module Etna
       return route && route.noauth?
     end
 
+    def janus_approved?(payload, token, request)
+      route = server.find_route(request)
+
+      # some routes don't need janus approval
+      return true if route && route.ignore_janus?
+
+      # only process task tokens right now
+      return true unless payload['task']
+
+      return false unless application.config(:janus) && application.config(:janus)[:host]
+
+      janus_client = Etna::Clients::Janus.new(
+        token: token,
+        host: application.config(:janus)[:host]
+      )
+
+      response = janus_client.validate_task_token()
+
+      return false unless response.code == '200'
+
+      return true
+    end
+
     def approve_user(request)
       token = request.cookies[application.config(:token_name)] || auth(request, :etna)
 
@@ -77,6 +100,8 @@ module Etna
 
       begin
         payload, header = application.sign.jwt_decode(token)
+
+        return false unless janus_approved?(payload, token, request)
         return request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
       rescue
         # bail out if anything goes wrong

data/lib/etna/client.rb

@@ -17,6 +17,13 @@ module Etna
 
     attr_reader :routes
 
+    def with_headers(headers, &block)
+      @request_headers = headers.compact
+      result = instance_eval(&block)
+      @request_headers = nil
+      return result
+    end
+
     def signed_route_path(route, params)
       path = route_path(route,params)
 
@@ -111,7 +118,7 @@ module Etna
 
     def body_request(type, endpoint, params = {}, &block)
       uri = request_uri(endpoint)
-      req = type.new(uri.request_uri, request_params)
+      req = type.new(uri.request_uri, request_headers)
       req.body = params.to_json
       request(uri, req, &block)
     end
@@ -124,7 +131,7 @@ module Etna
       else
         uri.query = URI.encode_www_form(params)
       end
-      req = type.new(uri.request_uri, request_params)
+      req = type.new(uri.request_uri, request_headers)
       request(uri, req, &block)
     end
 
@@ -132,12 +139,14 @@ module Etna
       URI("#{@host}#{endpoint}")
     end
 
-    def request_params
+    def request_headers
       {
           'Content-Type' => 'application/json',
           'Accept' => 'application/json, text/*',
           'Authorization' => "Etna #{@token}"
-      }
+      }.update(
+        @request_headers || {}
+      )
     end
 
     def status_check!(response)

data/lib/etna/clients/base_client.rb

@@ -8,10 +8,9 @@ module Etna
       attr_reader :host, :token, :ignore_ssl
       def initialize(host:, token:, ignore_ssl: false)
         raise "#{self.class.name} client configuration is missing host." unless host
-        raise "#{self.class.name} client configuration is missing token." unless token
 
         @token = token
-        raise "Your token is expired." if token_expired?
+        raise "Your token is expired." if token && token_expired?
 
         @etna_client = ::Etna::Client.new(
           host,
@@ -36,4 +35,4 @@ module Etna
       end
     end
   end
-end
+end

data/lib/etna/clients/janus.rb

@@ -1,2 +1,3 @@
 require_relative './janus/client'
 require_relative './janus/models'
+require_relative './janus/workflows'

data/lib/etna/clients/janus/client.rb

@@ -57,6 +57,25 @@ module Etna
 
         TokenResponse.new(token)
       end
+
+      def validate_task_token(validate_task_token_request = ValidateTaskTokenRequest.new)
+        token = nil
+        @etna_client.post('/api/tokens/task/validate', validate_task_token_request)
+      end
+
+      def get_nonce
+        @etna_client.get('/api/tokens/nonce').body
+      end
+
+      def generate_token(token_type, signed_nonce: nil, project_name: nil)
+        response = @etna_client.with_headers(
+          'Authorization' => signed_nonce ? "Signed-Nonce #{signed_nonce}" : nil
+        ) do
+          post('/api/tokens/generate', token_type: token_type, project_name: project_name)
+        end
+
+        response.body
+      end
     end
   end
 end

data/lib/etna/clients/janus/models.rb

@@ -51,6 +51,12 @@ module Etna
         end
       end
 
+      class ValidateTaskTokenRequest
+        def map
+          []
+        end
+      end
+
       class HtmlResponse
         attr_reader :raw
 
@@ -76,4 +82,4 @@ module Etna
       end
     end
   end
-end
+end

data/lib/etna/clients/janus/workflows.rb

@@ -0,0 +1 @@
+require_relative './workflows/generate_token_workflow'

data/lib/etna/clients/janus/workflows/generate_token_workflow.rb

@@ -0,0 +1,77 @@
+# Base workflow for setting up a project by a super user.
+# 1) Creates the project in janus
+# 2) Adds administrator(s) to Janus
+# 3) Refreshes the user's token with the new privileges.
+# 4) Creates the project in .
+
+require 'base64'
+require 'json'
+require 'ostruct'
+require_relative '../models'
+
+module Etna
+  module Clients
+    class Janus
+      class GenerateTokenWorkflow < Struct.new(:janus_client, :email, :project_name, :token_type, :private_key_file, keyword_init: true)
+        def generate!
+          nonce = janus_client.get_nonce
+
+          unless email
+            puts "Email address for #{janus_client.host} account?"
+            email = STDIN.gets.chomp
+          end
+
+          if use_nonce?
+            until private_key_file
+              puts "Location of private key file?"
+              private_key_file = ::File.expand_path(STDIN.gets.chomp)
+              unless File.exists?(private_key_file)
+                puts "No such file."
+                private_key_file = nil
+              end
+            end
+          end
+
+          if needs_project_name?
+            puts "Project name?"
+            project_name = STDIN.gets.chomp
+          end
+          
+          token = janus_client.generate_token(token_type, signed_nonce: use_nonce? ? signed_nonce(nonce) : nil, project_name: project_name)
+
+          puts token
+        end
+
+        private
+
+        def signed_nonce(nonce)
+          private_key = OpenSSL::PKey::RSA.new(File.read(private_key_file))
+
+          txt_to_sign = "#{nonce}.#{Base64.strict_encode64(email)}"
+
+          sig = Base64.strict_encode64(
+            private_key.sign(OpenSSL::Digest::SHA256.new,txt_to_sign)
+          )
+
+          "#{txt_to_sign}.#{sig}"
+        end
+
+        def use_nonce?
+          !task_token? || !janus_client.token
+        end
+
+        def needs_project_name?
+          task_token? && !project_name
+        end
+
+        def task_token?
+          token_type == 'task'
+        end
+
+        def user
+          @user ||= JSON.parse(Base64.urlsafe_decode64(magma_client.token.split('.')[1]))
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/magma/formatting/models_csv.rb

@@ -138,7 +138,7 @@ module Etna
                 attribute_type: attribute.attribute_type,
                 link_model_name: attribute.link_model_name,
                 reciprocal_link_type: models.find_reciprocal(model: model, attribute: attribute)&.attribute_type,
-                description: attribute.desc,
+                description: attribute.description,
                 display_name: attribute.display_name,
                 match: attribute.match,
                 format_hint: attribute.format_hint,
@@ -175,7 +175,7 @@ module Etna
           # This should line up with the attribute names _on the model itself_.
           ATTRIBUTE_ROW_ENTRIES = [
               :attribute_type,
-              :link_model_name, :desc,
+              :link_model_name, :description,
               :display_name, :format_hint,
               :restricted, :read_only,
               :validation, :attribute_group,
@@ -188,7 +188,6 @@ module Etna
 
           def format_row(row)
             replace_row_column(row, :attribute_type) { |s| AttributeType.new(s) }
-            replace_row_column(row, :desc) { row.delete(:description) }
             replace_row_column(row, :restricted, &COLUMN_AS_BOOLEAN)
             replace_row_column(row, :read_only, &COLUMN_AS_BOOLEAN)
             replace_row_column(row, :options) { |s| {"type" => "Array", "value" => s.split(',').map(&:strip)} }
@@ -250,7 +249,7 @@ module Etna
                 parent_att.name = parent_att.attribute_name = parent_model_name
                 parent_att.attribute_type = Etna::Clients::Magma::AttributeType::PARENT
                 parent_att.link_model_name = parent_model_name
-                parent_att.desc = prettify(parent_model_name)
+                parent_att.description = prettify(parent_model_name)
                 parent_att.display_name = prettify(parent_model_name)
               end
             end
@@ -275,7 +274,7 @@ module Etna
                   attr.attribute_name = attr.name = template.name
                   attr.attribute_type = parent_link_type
                   attr.link_model_name = template.name
-                  attr.desc = prettify(template.name)
+                  attr.description = prettify(template.name)
                   attr.display_name = prettify(template.name)
                 end
               end
@@ -338,7 +337,7 @@ module Etna
                     models.build_model(att.link_model_name).build_template.build_attributes.build_attribute(template.name).tap do |rec_att|
                       rec_att.attribute_name = rec_att.name = template.name
                       rec_att.display_name = prettify(template.name)
-                      rec_att.desc = prettify(template.name)
+                      rec_att.description = prettify(template.name)
                       rec_att.attribute_type = Etna::Clients::Magma::AttributeType::COLLECTION
                       rec_att.link_model_name = template.name
                     end

data/lib/etna/clients/magma/formatting/models_odm_xml.rb

@@ -246,7 +246,7 @@ module Etna
               }
 
               params['redcap:TextValidationType'] = redcap_text_validation_map[attribute_type] if redcap_text_validation_map[attribute_type]
-              params['redcap:FieldNote'] = attribute.desc if attribute.desc
+              params['redcap:FieldNote'] = attribute.description if attribute.description
               xml.ItemDef(params) do
                 xml.Question do
                   xml.send('TranslatedText', attribute_name.capitalize)

data/lib/etna/clients/magma/models.rb

@@ -10,7 +10,7 @@ require_relative '../base_client'
 module Etna
   module Clients
     class Magma < Etna::Clients::BaseClient
-      class RetrievalRequest < Struct.new(:model_name, :attribute_names, :record_names, :project_name, :page, :page_size, :order, :filter, keyword_init: true)
+      class RetrievalRequest < Struct.new(:model_name, :attribute_names, :record_names, :project_name, :page, :page_size, :order, :filter, :hide_templates, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**params)
@@ -18,7 +18,7 @@ module Etna
         end
       end
 
-      class QueryRequest < Struct.new(:query, :project_name, keyword_init: true)
+      class QueryRequest < Struct.new(:query, :project_name, :order, :page, :page_size, keyword_init: true)
         include JsonSerializableStruct
       end
 
@@ -117,14 +117,6 @@ module Etna
           super({action_name: 'update_attribute'}.update(args))
         end
 
-        def desc=(val)
-          self.description = val
-        end
-
-        def desc
-          self.description
-        end
-
         def as_json
           super(keep_nils: true)
         end
@@ -454,6 +446,16 @@ module Etna
           @raw = raw
         end
 
+        def is_edited?(other)
+          # Don't just override == in case need to do a full comparison.
+          editable_attribute_names = Attribute::EDITABLE_ATTRIBUTE_ATTRIBUTES.map(&:to_s)
+          
+          self_editable = raw.slice(*editable_attribute_names)
+          other_editable = other.raw.slice(*editable_attribute_names)
+
+          self_editable != other_editable
+        end
+
         # Sets certain attribute fields which are implicit, even when not set, to match server behavior.
         def set_field_defaults!
           @raw.replace({
@@ -511,12 +513,12 @@ module Etna
           raw['unique'] = val
         end
 
-        def desc
-          raw['desc']
+        def description
+          raw['description']
         end
 
-        def desc=(val)
-          @raw['desc'] = val
+        def description=(val)
+          @raw['description'] = val
         end
 
         def display_name
@@ -583,13 +585,10 @@ module Etna
           raw['options']
         end
 
-        # NOTE!  The Attribute class returns description as desc, where as actions take it in as description.
-        # There are shortcut methods that try to handle this on the action class side of things.  Ideally we would
-        # make this more consistent in the near future.
         COPYABLE_ATTRIBUTE_ATTRIBUTES = [
-            :attribute_name, :attribute_type, :desc, :display_name, :format_hint,
+            :attribute_name, :attribute_type, :display_name, :format_hint,
             :hidden, :link_model_name, :read_only, :attribute_group, :unique, :validation,
-            :restricted
+            :restricted, :description
         ]
 
         EDITABLE_ATTRIBUTE_ATTRIBUTES = UpdateAttributeAction.members & COPYABLE_ATTRIBUTE_ATTRIBUTES

data/lib/etna/clients/magma/workflows/crud_workflow.rb

@@ -48,8 +48,16 @@ module Etna
 
               retry
             rescue Etna::Error => e
-              raise e unless e.message.include?('not found')
-              break
+              if e.status === 502
+                if attempts > 5
+                  raise e
+                end
+
+                retry
+              else
+                raise e unless e.message.include?('not found')
+                break
+              end
             end
 
             documents += last_page.models.model(model_name).documents unless block_given?

data/lib/etna/clients/magma/workflows/file_linking_workflow.rb

@@ -95,7 +95,9 @@ module Etna
             file_path = ::File.dirname(file_path)
             {attribute_name => "https://metis.ucsf.edu/#{project_name}/browse/#{bucket_name}/#{file_path}"}
           else
-            {attribute_name => {path: "metis://#{project_name}/#{bucket_name}/#{file_path}"}}
+            {attribute_name => {
+              path: "metis://#{project_name}/#{bucket_name}/#{file_path}",
+              original_filename: File.basename(file_path)}}
           end
         end
 

data/lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

@@ -34,7 +34,7 @@ module Etna
               record_names,
               model_attributes_mask: model_attributes_mask,
               model_filters: model_filters,
-              page_size: 500,
+              page_size: 20,
           ) do |template, document|
             logger&.info("Materializing #{template.name}##{document[template.identifier]}")
             templates[template.name] = template

data/lib/etna/clients/magma/workflows/model_synchronization_workflow.rb

@@ -190,10 +190,7 @@ module Etna
             source_attribute = Attribute.new(source_attribute.raw)
             source_attribute.set_field_defaults!
 
-            source_editable = source_attribute.raw.slice(*Attribute::EDITABLE_ATTRIBUTE_ATTRIBUTES.map(&:to_s))
-            target_editable = target_attribute.raw.slice(*Attribute::EDITABLE_ATTRIBUTE_ATTRIBUTES.map(&:to_s))
-
-            if source_editable == target_editable
+            if !source_attribute.is_edited?(target_attribute)
               return
             end
 
@@ -226,7 +223,7 @@ module Etna
           if renames && (attribute_renames = renames[model_name]) && (new_name = attribute_renames[attribute_name])
             new_name = target_attribute_of_source(model_name, new_name)
 
-            unless target_model.template.attributes.include?(new_name)
+            unless target_model.template.attributes.attribute_keys.include?(new_name)
               if target_original_attribute
                 rename = RenameAttributeAction.new(model_name: target_model_name, attribute_name: target_attribute_name, new_attribute_name: new_name)
                 queue_update(rename)

data/lib/etna/clients/magma/workflows/update_attributes_from_csv_workflow.rb

@@ -241,7 +241,13 @@ module Etna
                 attribute_name_clean = attribute_name.strip
                 raise "Invalid attribute \"#{attribute_name_clean}\" for model #{@model_name}." unless attribute = @workflow.find_attribute(@model_name, attribute_name_clean)
 
-                attributes[attribute_name_clean] = stripped_value(attribute, @raw[attribute_name])
+                stripped = stripped_value(attribute, @raw[attribute_name])
+                
+                unless @workflow.hole_value.nil?
+                  next if stripped == @workflow.hole_value
+                end
+                
+                attributes[attribute_name_clean] = stripped
               end
             end
           end

data/lib/etna/clients/magma/workflows/walk_model_tree_workflow.rb

@@ -9,6 +9,50 @@ module Etna
       class WalkModelTreeWorkflow < Struct.new(:magma_crud, :logger, keyword_init: true)
         def initialize(**args)
           super(**({}.update(args)))
+          @template_for = {}
+        end
+
+        def all_attributes(template:)
+          template.attributes.attribute_keys
+        end
+
+        def safe_group_attributes(template:, attributes_mask:)
+          result = []
+
+          template.attributes.attribute_keys.each do |attribute_name|
+            next if template.attributes.attribute(attribute_name).attribute_type == Etna::Clients::Magma::AttributeType::TABLE
+            result << attribute_name if attribute_included?(attributes_mask, attribute_name)
+          end
+
+          result
+        end
+
+        def masked_attributes(template:, model_attributes_mask:, model_name:)
+          attributes_mask = model_attributes_mask[model_name]
+
+          if attributes_mask.nil?
+            return [
+              safe_group_attributes(template: template, attributes_mask: attributes_mask),
+              all_attributes(template: template)
+            ]
+          end
+
+          [(safe_group_attributes(template: template, attributes_mask: attributes_mask) + [template.identifier, 'parent']).uniq,
+            attributes_mask]
+        end
+
+        def attribute_included?(mask, attribute_name)
+          return true if mask.nil?
+          mask.include?(attribute_name)
+        end
+
+        def template_for(model_name)
+          @template_for[model_name] ||= magma_crud.magma_client.retrieve(RetrievalRequest.new(
+              project_name: magma_crud.project_name,
+              model_name: model_name,
+              record_names: [],
+              attribute_names: [],
+          )).models.model(model_name).template
         end
 
         def walk_from(
@@ -26,35 +70,43 @@ module Etna
             next if seen.include?([path[:from], model_name])
             seen.add([path[:from], model_name])
 
+            template = template_for(model_name)
+            query_attributes, walk_attributes = masked_attributes(
+              template: template,
+              model_attributes_mask: model_attributes_mask,
+              model_name: model_name
+            )
+
             request = RetrievalRequest.new(
                 project_name: magma_crud.project_name,
                 model_name: model_name,
                 record_names: path[:record_names],
                 filter: model_filters[model_name],
+                attribute_names: query_attributes,
                 page_size: page_size, page: 1
             )
 
             related_models = {}
 
             magma_crud.page_records(model_name, request) do |response|
-              model = response.models.model(model_name)
-              template = model.template
-
+              logger&.info("Fetched page of #{model_name}")
               tables = []
               collections = []
               links = []
               attributes = []
 
-              template.attributes.attribute_keys.each do |attr_name|
-                attributes_mask = model_attributes_mask[model_name]
-                black_listed = !attributes_mask.nil? && !attributes_mask.include?(attr_name)
-                next if black_listed && attr_name != template.identifier && attr_name != 'parent'
-                attributes << attr_name
+              model = response.models.model(model_name)
 
+              template.attributes.attribute_keys.each do |attr_name|
                 attr = template.attributes.attribute(attr_name)
-                if attr.attribute_type == AttributeType::TABLE
+                if attr.attribute_type == AttributeType::TABLE && attribute_included?(walk_attributes, attr_name)
                   tables << attr_name
-                elsif attr.attribute_type == AttributeType::COLLECTION
+                end
+
+                next unless attribute_included?(query_attributes, attr_name)
+                attributes << attr_name
+
+                if attr.attribute_type == AttributeType::COLLECTION
                   related_models[attr.link_model_name] ||= Set.new
                   collections << attr_name
                 elsif attr.attribute_type == AttributeType::LINK
@@ -63,20 +115,38 @@ module Etna
                 elsif attr.attribute_type == AttributeType::CHILD
                   related_models[attr.link_model_name] ||= Set.new
                   links << attr_name
-                elsif attr.attribute_type == AttributeType::PARENT && !black_listed
+                elsif attr.attribute_type == AttributeType::PARENT && attribute_included?(walk_attributes, attr_name)
                   related_models[attr.link_model_name] ||= Set.new
                   links << attr_name
                 end
               end
 
+              table_data = {}
+              # Request tables in an inner chunk.
+              tables.each do |table_attr|
+                request = RetrievalRequest.new(
+                  project_name: magma_crud.project_name,
+                  model_name: model_name,
+                  record_names: model.documents.document_keys,
+                  attribute_names: [table_attr],
+                )
+
+                logger&.info("Fetching inner table #{table_attr}...")
+
+                table_response = magma_crud.magma_client.retrieve(request)
+                d = table_response.models.model(model_name).documents
+                table_d = table_response.models.model(table_attr).documents
+                table_data[table_attr] = d.document_keys.map do |id|
+                  [id, d.document(id)[table_attr].map { |tid| table_d.document(tid) }]
+                end.to_h
+              end
+
               model.documents.document_keys.each do |key|
                 record = model.documents.document(key).slice(*attributes)
 
                 # Inline tables inside the record
                 tables.each do |table_attr|
-                  record[table_attr] = record[table_attr].map do |id|
-                    response.models.model(template.attributes.attribute(table_attr).link_model_name).documents.document(id)
-                  end unless record[table_attr].nil?
+                  record[table_attr] = table_data[table_attr][key] unless table_data[table_attr].nil?
                 end
 
                 collections.each do |collection_attr|

data/lib/etna/clients/metis/client.rb

@@ -69,6 +69,11 @@ module Etna
           @etna_client.folder_remove(delete_folder_request.to_h))
       end
 
+      def delete_file(delete_file_request)
+        FilesResponse.new(
+          @etna_client.file_remove(delete_file_request.to_h))
+      end
+
       def find(find_request)
         FoldersAndFilesResponse.new(
           @etna_client.bucket_find(find_request.to_h))

data/lib/etna/clients/metis/models.rb

@@ -95,6 +95,21 @@ module Etna
         end
       end
 
+      class DeleteFileRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
       class FindRequest < Struct.new(:project_name, :bucket_name, :limit, :offset, :params, keyword_init: true)
         include JsonSerializableStruct
 

data/lib/etna/command.rb

@@ -203,6 +203,7 @@ module Etna
       @subcommands ||= self.class.constants.sort.reduce({}) do |acc, n|
         acc.tap do
           c = self.class.const_get(n)
+          next unless c.respond_to?(:instance_methods)
           next unless c.instance_methods.include?(:find_command)
           v = c.new(self)
           acc[v.command_name] = v

data/lib/etna/directed_graph.rb

@@ -7,6 +7,59 @@ class DirectedGraph
   attr_reader :children
   attr_reader :parents
 
+  def full_parentage(n)
+    [].tap do |result|
+      q = @parents[n].keys.dup
+      seen = Set.new
+
+      until q.empty?
+        n = q.shift
+        next if seen.include?(n)
+        seen.add(n)
+
+        result << n
+        q.push(*@parents[n].keys)
+      end
+
+      result.uniq!
+    end
+  end
+
+  def as_normalized_hash(root, include_root = true)
+    q = [root]
+    {}.tap do |result|
+      if include_root
+        result[root] = []
+      end
+
+      seen = Set.new
+
+      until q.empty?
+        n = q.shift
+        next if seen.include?(n)
+        seen.add(n)
+
+        parentage = full_parentage(n)
+
+        @children[n].keys.each do |child_node|
+          q << child_node
+
+          if result.include?(n)
+            result[n] << child_node
+          end
+
+          parentage.each do |grandparent|
+            result[grandparent] << child_node if result.include?(grandparent)
+          end
+
+          result[child_node] = []
+        end
+      end
+
+      result.values.each(&:uniq!)
+    end
+  end
+
   def add_connection(parent, child)
     children = @children[parent] ||= {}
     child_children = @children[child] ||= {}
@@ -18,12 +71,13 @@ class DirectedGraph
     parents[parent] = parent_parents
   end
 
-  def serialized_path_from(root)
+  def serialized_path_from(root, include_root = true)
     seen = Set.new
     [].tap do |result|
-      result << root
+      result << root if include_root
       seen.add(root)
-      path_q = paths_from(root)
+      path_q = paths_from(root, include_root)
+      traversables = path_q.flatten
 
       until path_q.empty?
         next_path = path_q.shift
@@ -34,7 +88,7 @@ class DirectedGraph
           next if next_n.nil?
           next if seen.include?(next_n)
 
-          if @parents[next_n].keys.any? { |p| !seen.include?(p) }
+          if @parents[next_n].keys.any? { |p| !seen.include?(p) && traversables.include?(p) }
             next_path.unshift(next_n)
             path_q.push(next_path)
             break
@@ -47,9 +101,9 @@ class DirectedGraph
     end
   end
 
-  def paths_from(root)
+  def paths_from(root, include_root = true)
     [].tap do |result|
-      parents_of_map = descendants(root)
+      parents_of_map = descendants(root, include_root)
       seen = Set.new
 
       parents_of_map.to_a.sort_by { |k, parents| [-parents.length, k.inspect] }.each do |k, parents|
@@ -68,7 +122,7 @@ class DirectedGraph
     end
   end
 
-  def descendants(parent)
+  def descendants(parent, include_root = true)
     seen = Set.new
 
     seen.add(parent)
@@ -90,7 +144,7 @@ class DirectedGraph
     while child = queue.pop
       next if seen.include? child
       seen.add(child)
-      path = (paths[child] ||= [parent])
+      path = (paths[child] ||= (include_root ? [parent] : []))
 
       @children[child].keys.each do |child_child|
         queue.push child_child

data/lib/etna/route.rb

@@ -91,6 +91,10 @@ module Etna
       @auth && @auth[:noauth]
     end
 
+    def ignore_janus?
+      @auth && @auth[:ignore_janus]
+    end
+
     private
 
     def application

data/lib/etna/user.rb

@@ -46,7 +46,7 @@ module Etna
       viewer: /[Vv]/,
       restricted: /[AEV]/,
     }
-    def has_roles(project, *roles)
+    def has_any_role?(project, *roles)
       perm = permissions[project.to_s]
 
       return false unless perm
@@ -55,15 +55,23 @@ module Etna
     end
 
     def is_superuser? project=nil
-      has_roles(:administration, :admin)
+      has_any_role?(:administration, :admin)
+    end
+
+    def is_supereditor? project=nil
+      has_any_role?(:administration, :admin, :editor)
+    end
+
+    def is_superviewer? project=nil
+      has_any_role?(:administration, :admin, :editor, :viewer)
     end
 
     def can_edit? project
-      is_superuser? || has_roles(project, :admin, :editor)
+      is_supereditor? || has_any_role?(project, :admin, :editor)
     end
 
     def can_view? project
-      is_superuser? || has_roles(project, :admin, :editor, :viewer)
+      is_superviewer? || has_any_role?(project, :admin, :editor, :viewer)
     end
 
     # superusers - administrators of the Administration group - cannot
@@ -75,7 +83,7 @@ module Etna
     end
 
     def is_admin? project
-      is_superuser? || has_roles(project, :admin)
+      is_superuser? || has_any_role?(project, :admin)
     end
 
     def active? project=nil

data/lib/helpers.rb

@@ -57,9 +57,9 @@ module WithEtnaClients
         **EtnaApp.instance.config(:metis, environment) || {})
   end
 
-  def janus_client
+  def janus_client(opts={})
     @janus_client ||= Etna::Clients::Janus.new(
-        token: token,
+        token: opts.has_key?(:token) ? opts[:token] : token,
         ignore_ssl: EtnaApp.instance.config(:ignore_ssl),
         **EtnaApp.instance.config(:janus, environment) || {})
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.30
+  version: 0.1.35
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-03 00:00:00.000000000 Z
+date: 2021-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -94,20 +94,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: concurrent-ruby-ext
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: See summary
 email: Saurabh.Asthana@ucsf.edu
 executables:
@@ -131,6 +117,8 @@ files:
 - lib/etna/clients/janus.rb
 - lib/etna/clients/janus/client.rb
 - lib/etna/clients/janus/models.rb
+- lib/etna/clients/janus/workflows.rb
+- lib/etna/clients/janus/workflows/generate_token_workflow.rb
 - lib/etna/clients/magma.rb
 - lib/etna/clients/magma/client.rb
 - lib/etna/clients/magma/formatting.rb
@@ -213,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 2.7.6.3
 signing_key: 
 specification_version: 4
 summary: Base classes for Mount Etna applications