etna 0.1.20 → 0.1.25

data/lib/etna/clients/magma/workflows/walk_model_tree_workflow.rb

@@ -0,0 +1,101 @@
+require 'ostruct'
+require 'digest'
+require 'fileutils'
+require 'tempfile'
+
+module Etna
+  module Clients
+    class Magma
+      class WalkModelTreeWorkflow < Struct.new(:magma_crud, :logger, keyword_init: true)
+        def walk_from(
+            model_name,
+            record_names = 'all',
+            model_attributes_mask: {},
+            model_filters: {},
+            page_size: 100,
+            &block)
+          q = [ { model_name: model_name, from: nil, record_names: record_names } ]
+          seen = Set.new
+
+          while (path = q.pop)
+            model_name = path[:model_name]
+            next if seen.include?([path[:from], model_name])
+            seen.add([path[:from], model_name])
+
+            request = RetrievalRequest.new(
+                project_name: magma_crud.project_name,
+                model_name: model_name,
+                record_names: path[:record_names],
+                filter: model_filters[model_name],
+                page_size: page_size, page: 1
+            )
+
+            related_models = {}
+
+            magma_crud.page_records(model_name, request) do |response|
+              model = response.models.model(model_name)
+              template = model.template
+
+              tables = []
+              collections = []
+              links = []
+              attributes = []
+
+              template.attributes.attribute_keys.each do |attr_name|
+                attributes_mask = model_attributes_mask[model_name]
+                next if !attributes_mask.nil? && !attributes_mask.include?(attr_name) && attr_name != template.identifier
+                attributes << attr_name
+
+                attr = template.attributes.attribute(attr_name)
+                if attr.attribute_type == AttributeType::TABLE
+                  tables << attr_name
+                elsif attr.attribute_type == AttributeType::COLLECTION
+                  related_models[attr.link_model_name] ||= Set.new
+                  collections << attr_name
+                elsif attr.attribute_type == AttributeType::LINK
+                  related_models[attr.link_model_name] ||= Set.new
+                  links << attr_name
+                elsif attr.attribute_type == AttributeType::CHILD
+                  related_models[attr.link_model_name] ||= Set.new
+                  links << attr_name
+                elsif attr.attribute_type == AttributeType::PARENT
+                  related_models[attr.link_model_name] ||= Set.new
+                  links << attr_name
+                end
+              end
+
+              model.documents.document_keys.each do |key|
+                record = model.documents.document(key).slice(*attributes)
+
+                # Inline tables inside the record
+                tables.each do |table_attr|
+                  record[table_attr] = record[table_attr].map do |id|
+                    response.models.model(template.attributes.attribute(table_attr).link_model_name).documents.document(id)
+                  end unless record[table_attr].nil?
+                end
+
+                collections.each do |collection_attr|
+                  record[collection_attr].each do |collected_id|
+                    related_models[template.attributes.attribute(collection_attr).link_model_name].add(collected_id)
+                  end unless record[collection_attr].nil?
+                end
+
+                links.each do |link_attr|
+                  related_models[template.attributes.attribute(link_attr).link_model_name].add(record[link_attr]) unless record[link_attr].nil?
+                end
+
+                yield template, record
+              end
+            end
+
+            related_models.each do |link_model_name, id_set|
+              next if id_set.empty?
+              q.push({ model_name: link_model_name, from: model_name, record_names: id_set.to_a })
+            end
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/etna/clients/metis/client.rb

@@ -1,23 +1,19 @@
-require 'net/http/persistent'
 require 'net/http/post/multipart'
 require 'singleton'
 require 'cgi'
 require 'json'
 require_relative '../../client'
 require_relative './models'
+require_relative '../base_client'
 
 module Etna
   module Clients
-    class Metis
-      attr_reader :token
-      def initialize(host:, token:, persistent: true, ignore_ssl: false)
+    class Metis < Etna::Clients::BaseClient
+
+      def initialize(host:, token:, ignore_ssl: false)
         raise 'Metis client configuration is missing host.' unless host
         raise 'Metis client configuration is missing token.' unless token
-        @etna_client = ::Etna::Client.new(
-          host,
-          token,
-          persistent: persistent,
-          ignore_ssl: ignore_ssl)
+        @etna_client = ::Etna::Client.new(host, token, ignore_ssl: ignore_ssl)
 
         @token = token
       end
@@ -90,6 +86,23 @@ module Etna
         end
       end
 
+      def file_metadata(file_or_url = File.new)
+        if file_or_url.instance_of?(File)
+          download_path =  file_or_url.download_path
+        else
+          download_path = file_or_url.sub(%r!^https://[^/]*?/!, '/')
+        end
+
+        # Do not actually consume the data, however.
+        # TODO: implement HEAD requests in metis through apache.
+        @etna_client.get(download_path) do |response|
+          return {
+              etag: response['ETag'].gsub(/"/, ''),
+              size: response['Content-Length'],
+          }
+        end
+      end
+
       def upload_start(upload_start_request = UploadStartRequest.new)
         json = nil
         @etna_client.post(upload_start_request.upload_path, upload_start_request) do |res|

data/lib/etna/clients/metis/models.rb

@@ -1,9 +1,10 @@
-require_relative '../../json_serializable_struct'
 require 'ostruct'
+require_relative '../../json_serializable_struct'
+require_relative '../base_client'
 
 module Etna
   module Clients
-    class Metis
+    class Metis < Etna::Clients::BaseClient
       class ListFoldersRequest < Struct.new(:project_name, :bucket_name, :offset, :limit, keyword_init: true)
         include JsonSerializableStruct
 

data/lib/etna/clients/polyphemus/client.rb

@@ -1,23 +1,12 @@
-require 'net/http/persistent'
 require 'net/http/post/multipart'
 require 'singleton'
 require_relative '../../client'
 require_relative './models'
+require_relative '../base_client'
 
 module Etna
   module Clients
-    class Polyphemus
-      def initialize(host:, token:, ignore_ssl: false, persistent: true)
-        raise 'Polyphemus client configuration is missing host.' unless host
-        raise 'Polyphemus client configuration is missing token.' unless token
-        @etna_client = ::Etna::Client.new(
-          host,
-          token,
-          routes_available: false,
-          persistent: persistent,
-          ignore_ssl: ignore_ssl)
-      end
-
+    class Polyphemus < Etna::Clients::BaseClient
       def configuration(configuration_request = ConfigurationRequest.new)
         json = nil
         @etna_client.get(
@@ -28,6 +17,22 @@ module Etna
 
         ConfigurationResponse.new(json)
       end
+
+      def job(job_request = JobRequest.new)
+        # Because this is a streaming response, just yield the response back.
+        #   The consumer will have to iterate over the response.read_body, like
+        #
+        # polyphemus_client.job(request) do |response|
+        #   response.read_body do |fragment|
+        #     <fragment contains a chunk of text streamed back from the server>
+        #   end
+        # end
+        @etna_client.post(
+          "/#{job_request.project_name}/job",
+          job_request) do |res|
+            yield res
+        end
+      end
     end
   end
 end

data/lib/etna/clients/polyphemus/models.rb

@@ -1,17 +1,37 @@
 require 'ostruct'
 require_relative '../../json_serializable_struct'
+require_relative '../base_client'
 
 # TODO:  In the near future, I'd like to transition to specifying apis via SWAGGER and generating model stubs from the
 # common definitions.  For nowe I've written them out by hand here.
 module Etna
   module Clients
-    class Polyphemus
+    class Polyphemus < Etna::Clients::BaseClient
       class ConfigurationRequest
         def map
           []
         end
       end
 
+      class RedcapJobRequest < Struct.new(:model_names, :redcap_tokens, :commit, :project_name, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({model_names: 'all', commit: false}.update(params))
+        end
+
+        def to_json
+          {
+            job_type: Etna::Clients::Polyphemus::JobType::REDCAP,
+            job_params: {
+              commit: commit,
+              model_names: model_names,
+              redcap_tokens: redcap_tokens
+            }
+          }.to_json
+        end
+      end
+
       class ConfigurationResponse
         attr_reader :raw
 
@@ -63,6 +83,11 @@ module Etna
           @raw['auth_redirect']
         end
       end
+
+      class JobType < String
+        include Enum
+        REDCAP = JobType.new("redcap")
+      end
     end
   end
 end

data/lib/etna/csvs.rb

@@ -27,7 +27,8 @@ module Etna
         lineno += 1
         row = row.to_hash
         row.keys.each { |k| row[k].strip! if row[k] =~ /^\s+$/ } if @strip
-        row.select! { |k, v| !v.empty? } if @filter_empties
+        row.keys.each { |k| row[k] = "" if row[k].nil? } unless @filter_empties
+        row.select! { |k, v| !v.nil? && !v.empty? } if @filter_empties
         @row_formatter.call(row) unless @row_formatter.nil?
         yield row, lineno if block_given?
       end

data/lib/etna/filesystem.rb

@@ -0,0 +1,93 @@
+module Etna
+  # A class that encapsulates opening / reading file system entries that abstracts normal file access in order
+  # to make stubbing, substituting, and testing easier.
+  class Filesystem
+    def with_writeable(dest, opts = 'w', &block)
+      ::File.open(dest, opts, &block)
+    end
+
+    def with_readable(src, opts = 'r', &block)
+      ::File.open(src, opts, &block)
+    end
+
+    def mkdir_p(dir)
+      require 'fileutils'
+      ::FileUtils.mkdir_p(dir)
+    end
+
+    def rm_rf(dir)
+      require 'fileutils'
+      FileUtils.rm_rf(dir)
+    end
+
+    def tmpdir
+      ::Dir.tmpdir
+    end
+
+    def exist?(src)
+      ::File.exist?(src)
+    end
+
+    class EmptyIO < StringIO
+      def write(*args)
+        # Do nothing -- always leave empty
+      end
+    end
+
+    class Mock < Filesystem
+      def initialize(&new_io)
+        @files = {}
+        @dirs = {}
+        @new_io = new_io
+      end
+
+      def mkio(file, opts)
+        if @new_io.nil?
+          StringIO.new
+        else
+          @new_io.call(file, opts)
+        end
+      end
+
+      def with_writeable(dest, opts = 'w', &block)
+        if @dirs.include?(dest)
+          raise IOError.new("Path #{dest} is a directory")
+        end
+
+        dir, file = File.split(dest)
+        @dirs[dir] ||= Set.new
+        @dirs[dir].add(file)
+        yield (@files[dest] = mkio(dest, opts))
+      end
+
+      def mkdir_p(dest)
+        while !@dirs.include?(dest)
+          @dirs[dest] = Set.new
+          break if dest == "." || dest == "/"
+          dest, _ = File.split(dest)
+        end
+      end
+
+      def tmpdir
+        require 'securerandom'
+        "/tmp-#{SecureRandom::uuid}"
+      end
+
+      def with_readable(src, opts = 'r', &block)
+        if @dirs.include?(dest)
+          raise IOError.new("Path #{dest} is a directory")
+        end
+
+        if !@files.include?(dest)
+          raise IOError.new("Path #{dest} does not exist")
+        end
+
+        yield (@files[dest] ||= mkio(src, opts))
+      end
+
+      def exist?(src)
+        @files.include?(src) || @dirs.include?(src)
+      end
+    end
+  end
+end

data/lib/etna/spec/vcr.rb

@@ -30,6 +30,8 @@ def setup_base_vcr(spec_helper_dir)
       end
     end
 
+    # c.debug_logger = File.open('log/vcr_debug.log', 'w')
+
     c.default_cassette_options = {
         serialize_with: :compressed,
         record: if ENV['IS_CI'] == '1'
@@ -82,6 +84,11 @@ def setup_base_vcr(spec_helper_dir)
       end
     end
   end
+
+  require 'multipartable'
+  def Multipartable.secure_boundary
+    "--THIS-IS-STABLE-FOR-TESTING"
+  end
 end
 
 def prepare_vcr_secret

data/lib/etna/test_auth.rb

@@ -7,7 +7,7 @@ module Etna
   class TestAuth < Auth
     def self.token_header(params)
       token = Base64.strict_encode64(params.to_json)
-      return [ 'Authorization', "Etna #{token}" ]
+      return [ 'Authorization', "Etna something.#{token}" ]
     end
 
     def self.token_param(params)
@@ -36,8 +36,12 @@ module Etna
 
       return false unless token
 
-      # here we simply base64-encode our user hash and pass it through
-      payload = JSON.parse(Base64.decode64(token))
+      # Here we simply base64-encode our user hash and pass it through
+      # In order to behave more like "real" tokens, we expect the user hash to be
+      #   in index 1 after splitting by ".".
+      # We do this to support Metis client tests, we pass in tokens with multiple "."-separated parts, so
+      #   have to account for that.
+      payload = JSON.parse(Base64.decode64(token.split('.')[1]))
 
       request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
     end

data/lib/helpers.rb

@@ -6,6 +6,15 @@ module WithEtnaClients
     EtnaApp.instance.environment
   end
 
+  def exit(status=true)
+    WithEtnaClients.exit(status)
+  end
+
+  # Abstraction used to prevent accidental exist in specs.
+  def self.exit(status)
+    Kernel.exit(status)
+  end
+
   def token(ignore_environment: false)
     unless ignore_environment
       if environment == :many
@@ -38,9 +47,6 @@ module WithEtnaClients
     @magma_client ||= Etna::Clients::Magma.new(
         token: token,
         ignore_ssl: EtnaApp.instance.config(:ignore_ssl),
-        # Persistent connections cause problem with magma restarts, until we can fix that we should force them
-        # to close + reopen each request.
-        persistent: false,
         **EtnaApp.instance.config(:magma, environment) || {})
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.20
+  version: 0.1.25
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-14 00:00:00.000000000 Z
+date: 2021-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -98,6 +98,7 @@ files:
 - lib/etna/auth.rb
 - lib/etna/client.rb
 - lib/etna/clients.rb
+- lib/etna/clients/base_client.rb
 - lib/etna/clients/enum.rb
 - lib/etna/clients/janus.rb
 - lib/etna/clients/janus/client.rb
@@ -117,9 +118,11 @@ files:
 - lib/etna/clients/magma/workflows/file_linking_workflow.rb
 - lib/etna/clients/magma/workflows/json_converters.rb
 - lib/etna/clients/magma/workflows/json_validators.rb
+- lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb
 - lib/etna/clients/magma/workflows/model_synchronization_workflow.rb
 - lib/etna/clients/magma/workflows/record_synchronization_workflow.rb
 - lib/etna/clients/magma/workflows/update_attributes_from_csv_workflow.rb
+- lib/etna/clients/magma/workflows/walk_model_tree_workflow.rb
 - lib/etna/clients/metis.rb
 - lib/etna/clients/metis/client.rb
 - lib/etna/clients/metis/models.rb
@@ -140,6 +143,7 @@ files:
 - lib/etna/environment_scoped.rb
 - lib/etna/errors.rb
 - lib/etna/ext.rb
+- lib/etna/filesystem.rb
 - lib/etna/generate_autocompletion_script.rb
 - lib/etna/hmac.rb
 - lib/etna/json_serializable_struct.rb