etna 0.1.28 → 0.1.33

data/lib/etna/clients/magma/models.rb

@@ -519,6 +519,18 @@ module Etna
           @raw['desc'] = val
         end
 
+        # description and description= are needed
+        #   to make UpdateAttribute actions
+        #   work in the model_synchronization_workflow for
+        #   desc.
+        def description
+          raw['desc']
+        end
+
+        def description=(val)
+          @raw['desc'] = val
+        end
+
         def display_name
           raw['display_name']
         end
@@ -589,7 +601,7 @@ module Etna
         COPYABLE_ATTRIBUTE_ATTRIBUTES = [
             :attribute_name, :attribute_type, :desc, :display_name, :format_hint,
             :hidden, :link_model_name, :read_only, :attribute_group, :unique, :validation,
-            :restricted
+            :restricted, :description
         ]
 
         EDITABLE_ATTRIBUTE_ATTRIBUTES = UpdateAttributeAction.members & COPYABLE_ATTRIBUTE_ATTRIBUTES

data/lib/etna/clients/magma/workflows/create_project_workflow.rb

@@ -70,7 +70,7 @@ module Etna
           puts "Creating Janus project."
           create_janus_project!
           puts "Done! Adding you as an administrator on the project."
-          add_janus_user(user['email'], "#{user['first']} #{user['last']}", 'editor')
+          add_janus_user(user['email'], "#{user['name']}", 'editor')
           promote_to_administrator(user['email'])
           update_magma_client_token!
 

data/lib/etna/clients/magma/workflows/crud_workflow.rb

@@ -36,11 +36,28 @@ module Etna
           documents = Documents.new({})
           last_page = nil
           while last_page.nil? || last_page.models.model_keys.map { |k| last_page.models.model(k).documents.raw.length }.sum > 0
+            attempts = 0
             begin
+              attempts += 1
               last_page = magma_client.retrieve(request)
+            # Unfortunately, paging in magma is not great and times out from time to time.
+            rescue Net::ReadTimeout => e
+              if attempts > 5
+                raise e
+              end
+
+              retry
             rescue Etna::Error => e
-              raise e unless e.message.include?('not found')
-              break
+              if e.status === 502
+                if attempts > 5
+                  raise e
+                end
+
+                retry
+              else
+                raise e unless e.message.include?('not found')
+                break
+              end
             end
 
             documents += last_page.models.model(model_name).documents unless block_given?

data/lib/etna/clients/magma/workflows/file_linking_workflow.rb

@@ -95,7 +95,9 @@ module Etna
             file_path = ::File.dirname(file_path)
             {attribute_name => "https://metis.ucsf.edu/#{project_name}/browse/#{bucket_name}/#{file_path}"}
           else
-            {attribute_name => {path: "metis://#{project_name}/#{bucket_name}/#{file_path}"}}
+            {attribute_name => {
+              path: "metis://#{project_name}/#{bucket_name}/#{file_path}",
+              original_filename: File.basename(file_path)}}
           end
         end
 

data/lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

@@ -1,5 +1,4 @@
 require 'ostruct'
-require 'digest'
 require 'fileutils'
 require 'tempfile'
 
@@ -9,11 +8,11 @@ module Etna
       class MaterializeDataWorkflow < Struct.new(
           :metis_client, :magma_client, :project_name,
           :model_name, :model_filters, :model_attributes_mask,
-          :filesystem, :logger, :stub_files,
-          :skip_tmpdir, keyword_init: true)
+          :filesystem, :logger, :stub_files, :concurrency,
+          :record_names, keyword_init: true)
 
         def initialize(**kwds)
-          super(**({filesystem: Etna::Filesystem.new}.update(kwds)))
+          super(**({filesystem: Etna::Filesystem.new, concurrency: 10, record_names: "all"}.update(kwds)))
         end
 
         def magma_crud
@@ -25,31 +24,47 @@ module Etna
         end
 
         def materialize_all(dest)
-          tmpdir = skip_tmpdir ? nil : filesystem.tmpdir
+          templates = {}
+
+          semaphore = Concurrent::Semaphore.new(concurrency)
+          errors = Queue.new
+
+          model_walker.walk_from(
+              model_name,
+              record_names,
+              model_attributes_mask: model_attributes_mask,
+              model_filters: model_filters,
+              page_size: 20,
+          ) do |template, document|
+            logger&.info("Materializing #{template.name}##{document[template.identifier]}")
+            templates[template.name] = template
+
+            begin
+              if (error = errors.pop(true))
+                raise error
+              end
+            rescue ThreadError
+            end
 
-          begin
-            model_walker.walk_from(
-                model_name,
-                model_attributes_mask: model_attributes_mask,
-                model_filters: model_filters,
-            ) do |template, document|
-              logger&.info("Materializing #{template.name}##{document[template.identifier]}")
-              materialize_record(dest, tmpdir, template, document)
+            semaphore.acquire
+            Thread.new do
+              begin
+                materialize_record(dest, template, document)
+              rescue => e
+                errors << e
+              ensure
+                semaphore.release
+              end
             end
-          ensure
-            filesystem.rm_rf(tmpdir) unless skip_tmpdir
           end
-        end
 
-        def each_root_record
-          request = RetrievalRequest.new(project_name: project_name, model_name: model_name, record_names: "all",
-              filter: filter, page_size: 100, page: 1)
-          magma_crud.page_records(model_name, request) do |response|
-            model = response.models.model(model_name)
-            template = model.template
-            model.documents.document_keys.each do |key|
-              yield template, model.documents.document(key)
+          semaphore.acquire(concurrency)
+
+          begin
+            if (error = errors.pop(true))
+              raise error
             end
+          rescue ThreadError
           end
         end
 
@@ -78,11 +93,10 @@ module Etna
           @sync_metis_data_workflow ||= Etna::Clients::Metis::SyncMetisDataWorkflow.new(
               metis_client: metis_client,
               logger: logger,
-              skip_tmpdir: skip_tmpdir,
               filesystem: filesystem)
         end
 
-        def materialize_record(dest_dir, tmpdir, template, record)
+        def materialize_record(dest_dir, template, record)
           record_to_serialize = record.dup
 
           each_file(template, record) do |attr_name, url, filename, idx|
@@ -91,13 +105,14 @@ module Etna
             end
 
             dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: "_#{attr_name}_#{idx}#{File.extname(filename)}"))
-            sync_metis_data_workflow.copy_file(bin_root_dir: dest_dir, tmpdir: tmpdir, dest: dest_file, url: url, stub: stub_files)
-            record_to_serialize[attr_name] << { file: dest_file, original_filename: filename }
+            sync_metis_data_workflow.copy_file(dest: dest_file, url: url, stub: stub_files)
+            record_to_serialize[attr_name] << {file: dest_file, original_filename: filename}
           end
 
           dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: '.json'))
           filesystem.mkdir_p(File.dirname(dest_file))
           json = record_to_serialize.to_json
+
           filesystem.with_writeable(dest_file, "w", size_hint: json.bytes.length) do |io|
             io.write(json)
           end

data/lib/etna/clients/magma/workflows/model_synchronization_workflow.rb

@@ -226,7 +226,7 @@ module Etna
           if renames && (attribute_renames = renames[model_name]) && (new_name = attribute_renames[attribute_name])
             new_name = target_attribute_of_source(model_name, new_name)
 
-            unless target_model.template.attributes.include?(new_name)
+            unless target_model.template.attributes.attribute_keys.include?(new_name)
               if target_original_attribute
                 rename = RenameAttributeAction.new(model_name: target_model_name, attribute_name: target_attribute_name, new_attribute_name: new_name)
                 queue_update(rename)

data/lib/etna/clients/magma/workflows/update_attributes_from_csv_workflow.rb

@@ -37,8 +37,7 @@ module Etna
         end
 
         def update_attributes
-          method = json_values ? :update_json : :update
-          magma_crud.update_records(method: method) do |update_request|
+          magma_crud.update_records(method: :update_json) do |update_request|
             each_revision do |model_name, record_name, revision|
               update_request.update_revision(model_name, record_name, revision)
             end
@@ -53,10 +52,18 @@ module Etna
       end
 
       class RowBase
-        def stripped_value(attribute_value)
+        def attribute_is_json?(attribute)
+          [Etna::Clients::Magma::AttributeType::FILE,
+           Etna::Clients::Magma::AttributeType::FILE_COLLECTION,
+           Etna::Clients::Magma::AttributeType::IMAGE].include?(attribute.attribute_type)
+        end
+
+        def stripped_value(attribute, attribute_value)
           attribute_value = attribute_value&.strip
 
-          if attribute_value && @workflow.json_values && attribute_value != @workflow.hole_value
+          if attribute_value &&
+             ( @workflow.json_values || attribute_is_json?(attribute) ) &&
+             attribute_value != @workflow.hole_value
             attribute_value = JSON.parse(attribute_value)
           end
           attribute_value
@@ -123,7 +130,7 @@ module Etna
                     raise "Invalid attribute #{attribute_name} for model #{model_name}."
                   end
 
-                  stripped = stripped_value(@raw[index + 1])
+                  stripped = stripped_value(attribute, @raw[index + 1])
                   unless @workflow.hole_value.nil?
                     next if stripped == @workflow.hole_value
                   end
@@ -234,7 +241,13 @@ module Etna
                 attribute_name_clean = attribute_name.strip
                 raise "Invalid attribute \"#{attribute_name_clean}\" for model #{@model_name}." unless attribute = @workflow.find_attribute(@model_name, attribute_name_clean)
 
-                attributes[attribute_name_clean] = stripped_value(@raw[attribute_name])
+                stripped = stripped_value(attribute, @raw[attribute_name])
+                
+                unless @workflow.hole_value.nil?
+                  next if stripped == @workflow.hole_value
+                end
+                
+                attributes[attribute_name_clean] = stripped
               end
             end
           end

data/lib/etna/clients/magma/workflows/walk_model_tree_workflow.rb

@@ -7,6 +7,31 @@ module Etna
   module Clients
     class Magma
       class WalkModelTreeWorkflow < Struct.new(:magma_crud, :logger, keyword_init: true)
+        def initialize(**args)
+          super(**({}.update(args)))
+          @template_for = {}
+        end
+
+        def masked_attributes(template:, model_attributes_mask:, model_name:)
+          attributes_mask = model_attributes_mask[model_name]
+          return ["all", "all"] if attributes_mask.nil?
+          [(attributes_mask + [template.identifier, 'parent']).uniq, attributes_mask]
+        end
+
+        def attribute_included?(mask, attribute_name)
+          return true if mask == "all"
+          mask.include?(attribute_name)
+        end
+
+        def template_for(model_name)
+          @template_for[model_name] ||= magma_crud.magma_client.retrieve(RetrievalRequest.new(
+              project_name: magma_crud.project_name,
+              model_name: model_name,
+              record_names: [],
+              attribute_names: [],
+          )).models.model(model_name).template
+        end
+
         def walk_from(
             model_name,
             record_names = 'all',
@@ -22,28 +47,30 @@ module Etna
             next if seen.include?([path[:from], model_name])
             seen.add([path[:from], model_name])
 
+            template = template_for(model_name)
+            query_attributes, walk_attributes = masked_attributes(template: template, model_attributes_mask: model_attributes_mask, model_name: model_name)
+
             request = RetrievalRequest.new(
                 project_name: magma_crud.project_name,
                 model_name: model_name,
                 record_names: path[:record_names],
                 filter: model_filters[model_name],
+                attribute_names: query_attributes,
                 page_size: page_size, page: 1
             )
 
             related_models = {}
 
             magma_crud.page_records(model_name, request) do |response|
-              model = response.models.model(model_name)
-              template = model.template
-
               tables = []
               collections = []
               links = []
               attributes = []
 
+              model = response.models.model(model_name)
+
               template.attributes.attribute_keys.each do |attr_name|
-                attributes_mask = model_attributes_mask[model_name]
-                next if !attributes_mask.nil? && !attributes_mask.include?(attr_name) && attr_name != template.identifier
+                next unless attribute_included?(query_attributes, attr_name)
                 attributes << attr_name
 
                 attr = template.attributes.attribute(attr_name)
@@ -58,7 +85,7 @@ module Etna
                 elsif attr.attribute_type == AttributeType::CHILD
                   related_models[attr.link_model_name] ||= Set.new
                   links << attr_name
-                elsif attr.attribute_type == AttributeType::PARENT
+                elsif attr.attribute_type == AttributeType::PARENT && attribute_included?(walk_attributes, attr_name)
                   related_models[attr.link_model_name] ||= Set.new
                   links << attr_name
                 end

data/lib/etna/clients/metis/client.rb

@@ -69,6 +69,11 @@ module Etna
           @etna_client.folder_remove(delete_folder_request.to_h))
       end
 
+      def delete_file(delete_file_request)
+        FilesResponse.new(
+          @etna_client.file_remove(delete_file_request.to_h))
+      end
+
       def find(find_request)
         FoldersAndFilesResponse.new(
           @etna_client.bucket_find(find_request.to_h))
@@ -98,7 +103,7 @@ module Etna
         @etna_client.get(download_path) do |response|
           return {
               etag: response['ETag'].gsub(/"/, ''),
-              size: response['Content-Length'],
+              size: response['Content-Length'].to_i,
           }
         end
       end

data/lib/etna/clients/metis/models.rb

@@ -95,6 +95,21 @@ module Etna
         end
       end
 
+      class DeleteFileRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
       class FindRequest < Struct.new(:project_name, :bucket_name, :limit, :offset, :params, keyword_init: true)
         include JsonSerializableStruct
 

data/lib/etna/clients/metis/workflows/metis_download_workflow.rb

@@ -13,23 +13,27 @@ module Etna
         end
 
         # TODO: Might be possible to use range headers to select and resume downloads on failure in the future.
-        def do_download(dest_file, metis_file, &block)
+        def do_download(dest_file_or_io, metis_file, &block)
           size = metis_file.size
           completed = 0.0
           start = Time.now
 
-          ::File.open(dest_file, "w") do |io|
-            metis_client.download_file(metis_file) do |chunk|
-              io.write chunk
-              completed += chunk.size
-
-              block.call([
-                  :progress,
-                  size == 0 ? 1 : completed / size,
-                  (completed / (Time.now - start)).round(2),
-              ]) unless block.nil?
+          unless dest_file_or_io.is_a?(IO)
+            ::File.open(dest_file_or_io, 'w') do |io|
+              return do_download(dest_file_or_io, metis_file, &block)
             end
           end
+
+          metis_client.download_file(metis_file) do |chunk|
+            dest_file_or_io.write chunk
+            completed += chunk.size
+
+            block.call([
+                :progress,
+                size == 0 ? 1 : completed / size,
+                (completed / (Time.now - start)).round(2),
+            ]) unless block.nil?
+          end
         end
       end
     end

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -4,17 +4,26 @@ require 'fileutils'
 require 'tempfile'
 require 'securerandom'
 
+$digest_mutx = Mutex.new
+
 module Etna
   module Clients
     class Metis
       class MetisUploadWorkflow < Struct.new(:metis_client, :metis_uid, :project_name, :bucket_name, :max_attempts, keyword_init: true)
+        class StreamingUploadError < StandardError
+        end
+
 
         def initialize(args)
           super({max_attempts: 3, metis_uid: SecureRandom.hex}.update(args))
         end
 
-        def do_upload(source_file, dest_path, &block)
-          upload = Upload.new(source_file: source_file)
+        def do_upload(source_file_or_upload, dest_path, &block)
+          unless source_file_or_upload.is_a?(Upload)
+            upload = Upload.new(source_file: source_file_or_upload)
+          else
+            upload = source_file_or_upload
+          end
 
           dir = ::File.dirname(dest_path)
           metis_client.create_folder(CreateFolderRequest.new(
@@ -68,9 +77,11 @@ module Etna
 
               unsent_zero_byte_file = false
             rescue Etna::Error => e
-              m = yield [:error, e] unless block.nil?
-              if m == false
-                raise e
+              unless block.nil?
+                m = yield [:error, e]
+                if m == false
+                  raise e
+                end
               end
 
               if e.status == 422
@@ -90,13 +101,17 @@ module Etna
           end
         end
 
-        class Upload < Struct.new(:source_file, :next_blob_size, :current_byte_position, keyword_init: true)
+        class Upload
           INITIAL_BLOB_SIZE = 2 ** 10
           MAX_BLOB_SIZE = 2 ** 22
           ZERO_HASH = 'd41d8cd98f00b204e9800998ecf8427e'
 
-          def initialize(**args)
-            super
+          attr_accessor :source_file, :next_blob_size, :current_byte_position
+
+          def initialize(source_file: nil, next_blob_size: nil, current_byte_position: nil)
+            self.source_file = source_file
+            self.next_blob_size = next_blob_size
+            self.current_byte_position = current_byte_position
             self.next_blob_size = [file_size, INITIAL_BLOB_SIZE].min
             self.current_byte_position = 0
           end
@@ -108,10 +123,10 @@ module Etna
           def advance_position!
             self.current_byte_position = self.current_byte_position + self.next_blob_size
             self.next_blob_size = [
-                MAX_BLOB_SIZE,
-                # in fact we should stop when we hit the end of the file
-                file_size - current_byte_position
-            ].min
+                                      MAX_BLOB_SIZE,
+                                      # in fact we should stop when we hit the end of the file
+                                      file_size - current_byte_position
+                                  ].min
           end
 
           def complete?
@@ -119,7 +134,14 @@ module Etna
           end
 
           def next_blob_hash
-            Digest::MD5.hexdigest(next_blob_bytes)
+            bytes = next_blob_bytes
+            if bytes.empty?
+              return ZERO_HASH
+            end
+
+            $digest_mutx.synchronize do
+              return Digest::MD5.hexdigest(bytes)
+            end
           end
 
           def next_blob_bytes
@@ -131,6 +153,54 @@ module Etna
             self.next_blob_size = upload_response.next_blob_size
           end
         end
+
+        class StreamingIOUpload < Upload
+          def initialize(readable_io:, size_hint: 0, **args)
+            @readable_io = readable_io
+            @size_hint = size_hint
+            @read_position = 0
+            @last_bytes = ""
+            super(**args)
+          end
+
+          def file_size
+            @size_hint
+          end
+
+          def next_blob_bytes
+            next_left = current_byte_position
+            next_right = current_byte_position + next_blob_size
+
+            if next_right < @read_position
+              raise StreamingUploadError.new("Upload needs restart, but source is streaming and ephemeral. #{next_right} #{@read_position} You need to restart the source stream and create a new upload.")
+            elsif @read_position < next_left
+              # read from the stream and discard until we are positioned for the next read.
+              data = @readable_io.read(next_left - @read_position)
+              raise StreamingUploadError.new("Unexpected EOF in read stream") if data.nil?
+
+              @read_position += data.bytes.length
+            end
+
+            # If we have consumed all requested data, return what we have consumed.
+            # If we have requested no data, make sure to provide "" as the result.
+            if next_right == @read_position
+              return @last_bytes
+            end
+
+            if @read_position != next_left
+              raise StreamingUploadError.new("Alignment error, source data does not match expected upload resume. #{@read_position} #{next_left} Restart the upload to address.")
+            end
+
+            @last_bytes = "".tap do |bytes|
+              while @read_position < next_right
+                bytes << @readable_io.read(next_right - @read_position).tap do |data|
+                  raise StreamingUploadError.new("Unexpected EOF in read stream") if data.nil?
+                  @read_position += data.bytes.length
+                end
+              end
+            end
+          end
+        end
       end
     end
   end