etna 0.1.21 → 0.1.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 920faf7833d50d01c0a22d7326d0d3c454cdec27479ce81e4bf0b4012891f0bc
-  data.tar.gz: 189fad452f86a48ee19343eb7d5bd16bb9cf7308d8fd2dee41d042de2a76cc86
+  metadata.gz: 7075c5bf9637903a5aae31e83dbec99fc903945b362b5b454a3d31f95c03c6eb
+  data.tar.gz: 48e957e8286a613f7a1cafab2512976e625211782ae09a402053161af0f69b87
 SHA512:
-  metadata.gz: 64b2fef7604de4803790fea8543d39d8e6466456231ed014e30e633659049348274971c10a3231964664340cd13b616a2612c0cb0c89dd71337b4fb57f9432a5
-  data.tar.gz: 7ac816e004b1ace47c193f26944586d2eaca6e35a4c9fc7e76ab9dfe524b5f41cbf2f94abfa377006b80f59baaee1690b3582aa07f2a1ea851dc76fada707805
+  metadata.gz: 850effed7485f90957cb3ee53ebb97abaa3fe9c32d9de9ff16c5d30585b61f34ccf00bb88c702ee9f1271c8a883526cebc36218904c311c276d6de51aca0d1ba
+  data.tar.gz: 1c90072a96341cf75b869d08f9e1cd1e36c2a616873048d83c6c1cd2ffe57850c618745c80ee1e16e63c2c8c4c090cbb8c99ab19ed1541ec06d8b43d29824d0a

data/etna.completion

@@ -77,7 +77,7 @@ all_flag_completion_names="$all_flag_completion_names  "
 string_flag_completion_names="$string_flag_completion_names  "
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
-all_completion_names="apply_template attributes copy_template help"
+all_completion_names="apply_template attributes copy_template help load_from_redcap"
 all_completion_names="$all_completion_names $all_flag_completion_names"
 if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
 return
@@ -136,7 +136,7 @@ all_flag_completion_names="$all_flag_completion_names  "
 string_flag_completion_names="$string_flag_completion_names  "
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
-all_completion_names="create_file_linking_csv help update_from_csv"
+all_completion_names="create_file_linking_csv help load_table_from_csv update_from_csv"
 all_completion_names="$all_completion_names $all_flag_completion_names"
 if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
 return
@@ -252,6 +252,68 @@ return
 fi
 done
 return
+elif [[ "$1" == "load_table_from_csv" ]]; then
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__project_name__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__model_name__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__file_path__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+all_flag_completion_names="$all_flag_completion_names --execute "
+string_flag_completion_names="$string_flag_completion_names  "
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names=""
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+a=$1
+shift
+if [[ "$string_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
+return
 elif [[ "$1" == "update_from_csv" ]]; then
 shift
 if [[ "$#" == "1" ]];  then
@@ -417,6 +479,61 @@ return
 fi
 done
 return
+elif [[ "$1" == "load_from_redcap" ]]; then
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__project_name__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__redcap_tokens__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+all_flag_completion_names="$all_flag_completion_names --commit --models --record_names "
+string_flag_completion_names="$string_flag_completion_names --models --record_names "
+declare _completions_for_models="__models__"
+declare _completions_for_record_names="__record_names__"
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names=""
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+a=$1
+shift
+if [[ "$string_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
+return
 elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
 return
 elif [[ "$all_flag_completion_names" =~ $1\  ]]; then

data/lib/commands.rb

@@ -54,7 +54,7 @@ class EtnaApp
       boolean_flags << '--ignore-ssl'
 
       def execute(host, ignore_ssl: false)
-        polyphemus_client ||= Etna::Clients::Polyphemus.new(
+        polyphemus_client = Etna::Clients::Polyphemus.new(
             host: host,
             token: token(ignore_environment: true),
             ignore_ssl: ignore_ssl)
@@ -62,7 +62,7 @@ class EtnaApp
             polyphemus_client: polyphemus_client,
             config_file: EtnaApp.config_file_path)
         config = workflow.update_configuration_file(ignore_ssl: ignore_ssl)
-        logger.info("Updated #{config.environment} configuration from #{host}.")
+        logger&.info("Updated #{config.environment} configuration from #{host}.")
       end
 
       def setup(config)
@@ -299,6 +299,87 @@ class EtnaApp
             workflow.write_csv_io(filename: file)
           end
         end
+
+        class LoadTableFromCsv < Etna::Command
+          include WithEtnaClients
+
+          boolean_flags << '--execute'
+
+          def execute(project_name, model_name, file_path, execute: false)
+            request = Etna::Clients::Magma::RetrievalRequest.new(project_name: project_name)
+            request.model_name = model_name
+            request.attribute_names = 'all'
+            request.record_names = 'all'
+            model = magma_client.retrieve(request).models.model(model_name)
+            model_parent_name = model.template.attributes.all.select do |attribute|
+              attribute.attribute_type == Etna::Clients::Magma::AttributeType::PARENT
+            end.first.name
+
+            other_attribute_names = model.template.attributes.all.reject do |attribute|
+              attribute.attribute_type == Etna::Clients::Magma::AttributeType::PARENT
+            end.map do |attribute|
+              attribute.name
+            end
+
+            # NOTE: This does not call ensure_parent currently because of MVIR1 consent--
+            #   if the timepoint doesn't exist, the patient may be no study? (one example, at least)
+            update_request = Etna::Clients::Magma::UpdateRequest.new(project_name: project_name)
+
+            data = CSV.parse(File.read(file_path), headers: true)
+
+            data.by_row.each do |row|
+              revision = {}
+              other_attribute_names.each do |attribute_name|
+                revision[attribute_name] = row[attribute_name] unless row[attribute_name].nil?
+              end
+              update_request.append_table(model_parent_name, row[model_parent_name], model_name, revision)
+            end
+
+            puts update_request
+
+            if execute
+              magma_client.update_json(update_request)
+            end
+          end
+        end
+      end
+
+      class LoadFromRedcap < Etna::Command
+        include WithEtnaClients
+        include WithLogger
+        include StrongConfirmation
+
+        boolean_flags << '--commit'
+        string_flags << '--models'
+        string_flags << '--record_names'
+
+        def execute(project_name, redcap_tokens, models: "all", record_names: nil, commit: false)
+          raise "Must provide at least one REDCap token (comma-separated)." unless redcap_tokens.split(',').length > 0
+
+          puts "NOTE: This is a **preview** of what the data loading will look like. Use the --commit flag to load records into Magma." unless commit
+
+          polyphemus_client.job(Etna::Clients::Polyphemus::RedcapJobRequest.new(
+            model_names: "all" == models ? "all" : models.split(','),
+            record_names: nil == record_names || "existing" == record_names ? record_names : record_names.split(','),
+            redcap_tokens: redcap_tokens.split(','),
+            project_name: project_name,
+            commit: commit
+          )) do |response|
+            response.read_body do |chunk|
+              puts clean_sne_message(chunk)
+            end
+          end
+        end
+
+        def clean_sne_message(chunk)
+          chunk.split("\n").reject do |c|
+            c.start_with?("retry:") || c.start_with?("event:")
+          end.map do |c|
+            c.gsub("data:", "").strip
+          end.reject do |c|
+            c.empty?
+          end
+        end
       end
     end
   end

data/lib/etna.rb

@@ -18,6 +18,8 @@ require_relative './etna/spec'
 require_relative './etna/clients'
 require_relative './etna/csvs'
 require_relative './etna/environment_scoped'
+require_relative './etna/filesystem'
+require_relative './etna/formatting'
 
 class EtnaApp
   include Etna::Application

data/lib/etna/client.rb

@@ -1,14 +1,12 @@
-require 'net/http/persistent'
 require 'net/http/post/multipart'
 require 'singleton'
 require 'rack/utils'
 
 module Etna
   class Client
-    def initialize(host, token, routes_available: true, persistent: true, ignore_ssl: false)
+    def initialize(host, token, routes_available: true, ignore_ssl: false)
       @host = host.sub(%r!/$!, '')
       @token = token
-      @persistent = persistent
       @ignore_ssl = ignore_ssl
 
       if routes_available
@@ -38,6 +36,10 @@ module Etna
       query_request(Net::HTTP::Get, endpoint, params, &block)
     end
 
+    def head(endpoint, params = {}, &block)
+      query_request(Net::HTTP::Head, endpoint, params, &block)
+    end
+
     def options(endpoint, params = {}, &block)
       query_request(Net::HTTP::Options, endpoint, params, &block)
     end
@@ -79,16 +81,6 @@ module Etna
       end
     end
 
-    def persistent_connection
-      @http ||= begin
-        http = Net::HTTP::Persistent.new
-        http.read_timeout = 3600
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE if @ignore_ssl
-        http
-      end
-    end
-
-
     def body_request(type, endpoint, params = {}, &block)
       uri = request_uri(endpoint)
       req = type.new(uri.request_uri, request_params)
@@ -141,36 +133,23 @@ module Etna
 
     def request(uri, data)
       if block_given?
-        if @persistent
-          persistent_connection.request(uri, data) do |response|
+        verify_mode = @ignore_ssl ?
+          OpenSSL::SSL::VERIFY_NONE :
+          OpenSSL::SSL::VERIFY_PEER
+        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
+          http.request(data) do |response|
             status_check!(response)
             yield response
           end
-        else
-          verify_mode = @ignore_ssl ?
-            OpenSSL::SSL::VERIFY_NONE :
-            OpenSSL::SSL::VERIFY_PEER
-          Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
-            http.request(data) do |response|
-              status_check!(response)
-              yield response
-            end
-          end
         end
       else
-        if @persistent
-          response = persistent_connection.request(uri, data)
+        verify_mode = @ignore_ssl ?
+          OpenSSL::SSL::VERIFY_NONE :
+          OpenSSL::SSL::VERIFY_PEER
+        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
+          response = http.request(data)
           status_check!(response)
           return response
-        else
-          verify_mode = @ignore_ssl ?
-            OpenSSL::SSL::VERIFY_NONE :
-            OpenSSL::SSL::VERIFY_PEER
-          Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
-            response = http.request(data)
-            status_check!(response)
-            return response
-          end
         end
       end
     end

data/lib/etna/clients/base_client.rb

@@ -0,0 +1,39 @@
+require 'base64'
+require 'json'
+require 'date'
+
+module Etna
+  module Clients
+    class BaseClient
+      attr_reader :host, :token, :ignore_ssl
+      def initialize(host:, token:, ignore_ssl: false)
+        raise "#{self.class.name} client configuration is missing host." unless host
+        raise "#{self.class.name} client configuration is missing token." unless token
+
+        @token = token
+        raise "Your token is expired." if token_expired?
+
+        @etna_client = ::Etna::Client.new(
+          host,
+          token,
+          routes_available: false,
+          ignore_ssl: ignore_ssl)
+        @host = host
+        @ignore_ssl = ignore_ssl
+      end
+
+      def token_expired?
+        # Has the token already expired?
+        token_will_expire?(0)
+      end
+
+      def token_will_expire?(offset=3000)
+        # offset in seconds
+        # Will the user's token expire in the given amount of time?
+        epoch_seconds = JSON.parse(Base64.urlsafe_decode64(token.split('.')[1]))["exp"]
+        expiration = DateTime.strptime(epoch_seconds.to_s, "%s")
+        expiration <= DateTime.now.new_offset + offset
+      end
+    end
+  end
+end

data/lib/etna/clients/janus/client.rb

@@ -1,23 +1,12 @@
-require 'net/http/persistent'
 require 'net/http/post/multipart'
 require 'singleton'
 require_relative '../../client'
 require_relative './models'
+require_relative '../base_client'
 
 module Etna
   module Clients
-    class Janus
-      def initialize(host:, token:, persistent: true, ignore_ssl: false)
-        raise 'Janus client configuration is missing host.' unless host
-        raise 'Janus client configuration is missing token.' unless token
-        @etna_client = ::Etna::Client.new(
-          host,
-          token,
-          routes_available: false,
-          persistent: persistent,
-          ignore_ssl: ignore_ssl)
-      end
-
+    class Janus < Etna::Clients::BaseClient
       def get_project(get_project_request = GetProjectRequest.new)
         html = nil
         @etna_client.get(

data/lib/etna/clients/janus/models.rb

@@ -1,11 +1,12 @@
 require 'ostruct'
 require_relative '../../json_serializable_struct'
+require_relative '../base_client'
 
 # TODO:  In the near future, I'd like to transition to specifying apis via SWAGGER and generating model stubs from the
 # common definitions.  For nowe I've written them out by hand here.
 module Etna
   module Clients
-    class Janus
+    class Janus < Etna::Clients::BaseClient
       class GetProjectRequest < Struct.new(:project_name, keyword_init: true)
         include JsonSerializableStruct
 

data/lib/etna/clients/magma/client.rb

@@ -1,27 +1,12 @@
-require 'net/http/persistent'
 require 'net/http/post/multipart'
 require 'singleton'
+require_relative '../base_client'
 require_relative '../../client'
 require_relative './models'
 
 module Etna
   module Clients
-    class Magma
-      attr_reader :host, :token, :ignore_ssl
-      def initialize(host:, token:, persistent: true, ignore_ssl: false)
-        raise 'Magma client configuration is missing host.' unless host
-        raise 'Magma client configuration is missing token.' unless token
-        @etna_client = ::Etna::Client.new(
-          host,
-          token,
-          routes_available: false,
-          persistent: persistent,
-          ignore_ssl: ignore_ssl)
-        @host = host
-        @token = token
-        @ignore_ssl = ignore_ssl
-      end
-
+    class Magma < Etna::Clients::BaseClient
       # This endpoint returns models and records by name:
       # e.g. params:
       # {