etna 0.1.27 → 0.1.32

data/lib/etna/clients/magma/workflows/model_synchronization_workflow.rb

@@ -226,7 +226,7 @@ module Etna
           if renames && (attribute_renames = renames[model_name]) && (new_name = attribute_renames[attribute_name])
             new_name = target_attribute_of_source(model_name, new_name)
 
-            unless target_model.template.attributes.include?(new_name)
+            unless target_model.template.attributes.attribute_keys.include?(new_name)
               if target_original_attribute
                 rename = RenameAttributeAction.new(model_name: target_model_name, attribute_name: target_attribute_name, new_attribute_name: new_name)
                 queue_update(rename)

data/lib/etna/clients/magma/workflows/update_attributes_from_csv_workflow.rb

@@ -37,8 +37,7 @@ module Etna
         end
 
         def update_attributes
-          method = json_values ? :update_json : :update
-          magma_crud.update_records(method: method) do |update_request|
+          magma_crud.update_records(method: :update_json) do |update_request|
             each_revision do |model_name, record_name, revision|
               update_request.update_revision(model_name, record_name, revision)
             end
@@ -53,10 +52,18 @@ module Etna
       end
 
       class RowBase
-        def stripped_value(attribute_value)
+        def attribute_is_json?(attribute)
+          [Etna::Clients::Magma::AttributeType::FILE,
+           Etna::Clients::Magma::AttributeType::FILE_COLLECTION,
+           Etna::Clients::Magma::AttributeType::IMAGE].include?(attribute.attribute_type)
+        end
+
+        def stripped_value(attribute, attribute_value)
           attribute_value = attribute_value&.strip
 
-          if attribute_value && @workflow.json_values && attribute_value != @workflow.hole_value
+          if attribute_value &&
+             ( @workflow.json_values || attribute_is_json?(attribute) ) &&
+             attribute_value != @workflow.hole_value
             attribute_value = JSON.parse(attribute_value)
           end
           attribute_value
@@ -123,7 +130,7 @@ module Etna
                     raise "Invalid attribute #{attribute_name} for model #{model_name}."
                   end
 
-                  stripped = stripped_value(@raw[index + 1])
+                  stripped = stripped_value(attribute, @raw[index + 1])
                   unless @workflow.hole_value.nil?
                     next if stripped == @workflow.hole_value
                   end
@@ -234,7 +241,13 @@ module Etna
                 attribute_name_clean = attribute_name.strip
                 raise "Invalid attribute \"#{attribute_name_clean}\" for model #{@model_name}." unless attribute = @workflow.find_attribute(@model_name, attribute_name_clean)
 
-                attributes[attribute_name_clean] = stripped_value(@raw[attribute_name])
+                stripped = stripped_value(attribute, @raw[attribute_name])
+                
+                unless @workflow.hole_value.nil?
+                  next if stripped == @workflow.hole_value
+                end
+                
+                attributes[attribute_name_clean] = stripped
               end
             end
           end

data/lib/etna/clients/magma/workflows/walk_model_tree_workflow.rb

@@ -7,6 +7,31 @@ module Etna
   module Clients
     class Magma
       class WalkModelTreeWorkflow < Struct.new(:magma_crud, :logger, keyword_init: true)
+        def initialize(**args)
+          super(**({}.update(args)))
+          @template_for = {}
+        end
+
+        def masked_attributes(template:, model_attributes_mask:, model_name:)
+          attributes_mask = model_attributes_mask[model_name]
+          return ["all", "all"] if attributes_mask.nil?
+          [(attributes_mask + [template.identifier, 'parent']).uniq, attributes_mask]
+        end
+
+        def attribute_included?(mask, attribute_name)
+          return true if mask == "all"
+          mask.include?(attribute_name)
+        end
+
+        def template_for(model_name)
+          @template_for[model_name] ||= magma_crud.magma_client.retrieve(RetrievalRequest.new(
+              project_name: magma_crud.project_name,
+              model_name: model_name,
+              record_names: [],
+              attribute_names: [],
+          )).models.model(model_name).template
+        end
+
         def walk_from(
             model_name,
             record_names = 'all',
@@ -22,28 +47,30 @@ module Etna
             next if seen.include?([path[:from], model_name])
             seen.add([path[:from], model_name])
 
+            template = template_for(model_name)
+            query_attributes, walk_attributes = masked_attributes(template: template, model_attributes_mask: model_attributes_mask, model_name: model_name)
+
             request = RetrievalRequest.new(
                 project_name: magma_crud.project_name,
                 model_name: model_name,
                 record_names: path[:record_names],
                 filter: model_filters[model_name],
+                attribute_names: query_attributes,
                 page_size: page_size, page: 1
             )
 
             related_models = {}
 
             magma_crud.page_records(model_name, request) do |response|
-              model = response.models.model(model_name)
-              template = model.template
-
               tables = []
               collections = []
               links = []
               attributes = []
 
+              model = response.models.model(model_name)
+
               template.attributes.attribute_keys.each do |attr_name|
-                attributes_mask = model_attributes_mask[model_name]
-                next if !attributes_mask.nil? && !attributes_mask.include?(attr_name) && attr_name != template.identifier
+                next unless attribute_included?(query_attributes, attr_name)
                 attributes << attr_name
 
                 attr = template.attributes.attribute(attr_name)
@@ -58,7 +85,7 @@ module Etna
                 elsif attr.attribute_type == AttributeType::CHILD
                   related_models[attr.link_model_name] ||= Set.new
                   links << attr_name
-                elsif attr.attribute_type == AttributeType::PARENT
+                elsif attr.attribute_type == AttributeType::PARENT && attribute_included?(walk_attributes, attr_name)
                   related_models[attr.link_model_name] ||= Set.new
                   links << attr_name
                 end

data/lib/etna/clients/metis/client.rb

@@ -69,6 +69,11 @@ module Etna
           @etna_client.folder_remove(delete_folder_request.to_h))
       end
 
+      def delete_file(delete_file_request)
+        FilesResponse.new(
+          @etna_client.file_remove(delete_file_request.to_h))
+      end
+
       def find(find_request)
         FoldersAndFilesResponse.new(
           @etna_client.bucket_find(find_request.to_h))
@@ -98,7 +103,7 @@ module Etna
         @etna_client.get(download_path) do |response|
           return {
               etag: response['ETag'].gsub(/"/, ''),
-              size: response['Content-Length'],
+              size: response['Content-Length'].to_i,
           }
         end
       end

data/lib/etna/clients/metis/models.rb

@@ -95,6 +95,21 @@ module Etna
         end
       end
 
+      class DeleteFileRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
       class FindRequest < Struct.new(:project_name, :bucket_name, :limit, :offset, :params, keyword_init: true)
         include JsonSerializableStruct
 

data/lib/etna/clients/metis/workflows/metis_download_workflow.rb

@@ -13,23 +13,27 @@ module Etna
         end
 
         # TODO: Might be possible to use range headers to select and resume downloads on failure in the future.
-        def do_download(dest_file, metis_file, &block)
+        def do_download(dest_file_or_io, metis_file, &block)
           size = metis_file.size
           completed = 0.0
           start = Time.now
 
-          ::File.open(dest_file, "w") do |io|
-            metis_client.download_file(metis_file) do |chunk|
-              io.write chunk
-              completed += chunk.size
-
-              block.call([
-                  :progress,
-                  size == 0 ? 1 : completed / size,
-                  (completed / (Time.now - start)).round(2),
-              ]) unless block.nil?
+          unless dest_file_or_io.is_a?(IO)
+            ::File.open(dest_file_or_io, 'w') do |io|
+              return do_download(dest_file_or_io, metis_file, &block)
             end
           end
+
+          metis_client.download_file(metis_file) do |chunk|
+            dest_file_or_io.write chunk
+            completed += chunk.size
+
+            block.call([
+                :progress,
+                size == 0 ? 1 : completed / size,
+                (completed / (Time.now - start)).round(2),
+            ]) unless block.nil?
+          end
         end
       end
     end

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -4,17 +4,26 @@ require 'fileutils'
 require 'tempfile'
 require 'securerandom'
 
+$digest_mutx = Mutex.new
+
 module Etna
   module Clients
     class Metis
       class MetisUploadWorkflow < Struct.new(:metis_client, :metis_uid, :project_name, :bucket_name, :max_attempts, keyword_init: true)
+        class StreamingUploadError < StandardError
+        end
+
 
         def initialize(args)
           super({max_attempts: 3, metis_uid: SecureRandom.hex}.update(args))
         end
 
-        def do_upload(source_file, dest_path, &block)
-          upload = Upload.new(source_file: source_file)
+        def do_upload(source_file_or_upload, dest_path, &block)
+          unless source_file_or_upload.is_a?(Upload)
+            upload = Upload.new(source_file: source_file_or_upload)
+          else
+            upload = source_file_or_upload
+          end
 
           dir = ::File.dirname(dest_path)
           metis_client.create_folder(CreateFolderRequest.new(
@@ -68,9 +77,11 @@ module Etna
 
               unsent_zero_byte_file = false
             rescue Etna::Error => e
-              m = yield [:error, e] unless block.nil?
-              if m == false
-                raise e
+              unless block.nil?
+                m = yield [:error, e]
+                if m == false
+                  raise e
+                end
               end
 
               if e.status == 422
@@ -90,13 +101,17 @@ module Etna
           end
         end
 
-        class Upload < Struct.new(:source_file, :next_blob_size, :current_byte_position, keyword_init: true)
+        class Upload
           INITIAL_BLOB_SIZE = 2 ** 10
           MAX_BLOB_SIZE = 2 ** 22
           ZERO_HASH = 'd41d8cd98f00b204e9800998ecf8427e'
 
-          def initialize(**args)
-            super
+          attr_accessor :source_file, :next_blob_size, :current_byte_position
+
+          def initialize(source_file: nil, next_blob_size: nil, current_byte_position: nil)
+            self.source_file = source_file
+            self.next_blob_size = next_blob_size
+            self.current_byte_position = current_byte_position
             self.next_blob_size = [file_size, INITIAL_BLOB_SIZE].min
             self.current_byte_position = 0
           end
@@ -108,10 +123,10 @@ module Etna
           def advance_position!
             self.current_byte_position = self.current_byte_position + self.next_blob_size
             self.next_blob_size = [
-                MAX_BLOB_SIZE,
-                # in fact we should stop when we hit the end of the file
-                file_size - current_byte_position
-            ].min
+                                      MAX_BLOB_SIZE,
+                                      # in fact we should stop when we hit the end of the file
+                                      file_size - current_byte_position
+                                  ].min
           end
 
           def complete?
@@ -119,7 +134,14 @@ module Etna
           end
 
           def next_blob_hash
-            Digest::MD5.hexdigest(next_blob_bytes)
+            bytes = next_blob_bytes
+            if bytes.empty?
+              return ZERO_HASH
+            end
+
+            $digest_mutx.synchronize do
+              return Digest::MD5.hexdigest(bytes)
+            end
           end
 
           def next_blob_bytes
@@ -131,6 +153,54 @@ module Etna
             self.next_blob_size = upload_response.next_blob_size
           end
         end
+
+        class StreamingIOUpload < Upload
+          def initialize(readable_io:, size_hint: 0, **args)
+            @readable_io = readable_io
+            @size_hint = size_hint
+            @read_position = 0
+            @last_bytes = ""
+            super(**args)
+          end
+
+          def file_size
+            @size_hint
+          end
+
+          def next_blob_bytes
+            next_left = current_byte_position
+            next_right = current_byte_position + next_blob_size
+
+            if next_right < @read_position
+              raise StreamingUploadError.new("Upload needs restart, but source is streaming and ephemeral. #{next_right} #{@read_position} You need to restart the source stream and create a new upload.")
+            elsif @read_position < next_left
+              # read from the stream and discard until we are positioned for the next read.
+              data = @readable_io.read(next_left - @read_position)
+              raise StreamingUploadError.new("Unexpected EOF in read stream") if data.nil?
+
+              @read_position += data.bytes.length
+            end
+
+            # If we have consumed all requested data, return what we have consumed.
+            # If we have requested no data, make sure to provide "" as the result.
+            if next_right == @read_position
+              return @last_bytes
+            end
+
+            if @read_position != next_left
+              raise StreamingUploadError.new("Alignment error, source data does not match expected upload resume. #{@read_position} #{next_left} Restart the upload to address.")
+            end
+
+            @last_bytes = "".tap do |bytes|
+              while @read_position < next_right
+                bytes << @readable_io.read(next_right - @read_position).tap do |data|
+                  raise StreamingUploadError.new("Unexpected EOF in read stream") if data.nil?
+                  @read_position += data.bytes.length
+                end
+              end
+            end
+          end
+        end
       end
     end
   end

data/lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb

@@ -1,107 +1,71 @@
 require 'ostruct'
-require 'digest'
 require 'fileutils'
 require 'tempfile'
 
 module Etna
   module Clients
     class Metis
-      class SyncMetisDataWorkflow < Struct.new(:metis_client, :filesystem, :project_name, :bucket_name,
-          :logger, :skip_tmpdir, keyword_init: true)
-        def copy_directory(src, dest, root = dest, tmpdir = nil)
-          own_tmpdir = tmpdir.nil? && !skip_tmpdir
-          if own_tmpdir
-            tmpdir = filesystem.tmpdir
-          end
-
-          begin
-            response = metis_client.list_folder(ListFolderRequest.new(project_name: project_name, bucket_name: bucket_name, folder_path: src))
+      class SyncMetisDataWorkflow < Struct.new(:metis_client, :filesystem, :project_name, :bucket_name, :logger, keyword_init: true)
+        def copy_directory(src, dest, root = dest)
+          response = metis_client.list_folder(ListFolderRequest.new(project_name: project_name, bucket_name: bucket_name, folder_path: src))
 
-            response.files.all.each do |file|
-              logger&.info("Copying file #{file.file_path} (#{Etna::Formatting.as_size(file.size)})")
-              copy_file(bin_root_dir: root, tmpdir: tmpdir, dest: ::File.join(dest, file.file_name), url: file.download_url)
-            end
-
-            response.folders.all.each do |folder|
-              copy_directory(::File.join(src, folder.folder_name), ::File.join(dest, folder.folder_name), root, tmpdir)
-            end
-          ensure
-            filesystem.rm_rf(tmpdir) if own_tmpdir
+          response.files.all.each do |file|
+            logger&.info("Copying file #{file.file_path} (#{Etna::Formatting.as_size(file.size)})")
+            copy_file(dest: ::File.join(dest, file.file_name), url: file.download_url)
           end
-        end
-
-        def bin_file_name(etag:)
-          "bin/#{etag}"
-        end
 
-        def with_maybe_intermediate_tmp_dest(bin_file_name:, tmpdir:, dest_file_name:, &block)
-          filesystem.mkdir_p(::File.dirname(dest_file_name))
-          if tmpdir.nil?
-            yield dest_file_name
-          else
-            tmp_file = ::File.join(tmpdir, ::File.basename(bin_file_name))
-            yield tmp_file
-            filesystem.mv(tmp_file, dest_file_name)
+          response.folders.all.each do |folder|
+            copy_directory(::File.join(src, folder.folder_name), ::File.join(dest, folder.folder_name), root)
           end
         end
 
-        def copy_file(bin_root_dir:, tmpdir:, dest:, url:, stub: false)
+        def copy_file(dest:, url:, stub: false)
           metadata = metis_client.file_metadata(url)
-          etag = metadata[:etag]
           size = metadata[:size]
 
-          dest_bin_file = ::File.join(bin_root_dir, bin_file_name(etag: etag))
-          # Already materialized, continue
-          if filesystem.exist?(dest_bin_file)
-            return
-          end
-
-          with_maybe_intermediate_tmp_dest(bin_file_name: dest_bin_file, tmpdir: tmpdir, dest_file_name: dest) do |tmp_file|
-            upload_timings = []
-            upload_amount = 0
-            last_rate = 0.00001
-
-            filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
-              if stub
-                io.write("(stub) #{size} bytes")
-              else
-                metis_client.download_file(url) do |chunk|
-                  io.write(chunk)
-
-                  upload_timings << [chunk.length, Time.now.to_f]
-                  upload_amount += chunk.length
-
-                  if upload_timings.length > 150
-                    s, _ = upload_timings.shift
-                    upload_amount -= s
-                  end
+          tmp_file = dest
+          upload_timings = []
+          upload_amount = 0
+          last_rate = 0.00001
+          remaining = size
+
+          filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
+            if stub
+              io.write("(stub) #{size} bytes")
+            else
+              metis_client.download_file(url) do |chunk|
+                io.write(chunk)
+
+                upload_timings << [chunk.length, Time.now.to_f]
+                upload_amount += chunk.length
+                remaining -= chunk.length
+
+                if upload_timings.length > 150
+                  s, _ = upload_timings.shift
+                  upload_amount -= s
+                end
 
-                  _, start_time = upload_timings.first
-                  _, end_time = upload_timings.last
+                _, start_time = upload_timings.first
+                _, end_time = upload_timings.last
 
-                  if start_time == end_time
-                    next
-                  end
+                if start_time == end_time
+                  next
+                end
 
-                  rate = upload_amount / (end_time - start_time)
+                rate = upload_amount / (end_time - start_time)
 
-                  if rate / last_rate > 1.3 || rate / last_rate < 0.7
-                    logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second")
+                if rate / last_rate > 1.3 || rate / last_rate < 0.7
+                  logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second, #{Etna::Formatting.as_size(remaining)} remaining")
 
-                    if rate == 0
-                      last_rate = 0.0001
-                    else
-                      last_rate = rate
-                    end
+                  if rate == 0
+                    last_rate = 0.0001
+                  else
+                    last_rate = rate
                   end
                 end
               end
-            end
-          end
 
-          filesystem.mkdir_p(::File.dirname(dest_bin_file))
-          filesystem.with_writeable(dest_bin_file, 'w', size_hint: 0) do |io|
-            # empty file marking that this etag has been moved, to save a future write.
+            end
           end
         end
       end