etna 0.1.51 → 0.2.0

data/lib/etna/clients/metis/models.rb

@@ -158,6 +158,21 @@ module Etna
         end
       end
 
+      class TailBucketRequest < Struct.new(:project_name, :bucket_name, :type, :folder_id, :batch_start, :batch_end, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
       class FindRequest < Struct.new(:project_name, :bucket_name, :limit, :offset, :params, :hide_paths, keyword_init: true)
         include JsonSerializableStruct
 
@@ -219,25 +234,99 @@ module Etna
         end
       end
 
-      class FoldersResponse
-        attr_reader :raw
+      class TailNode < Etna::Clients::Response
+        def initialize(tail, raw = {})
+          super(raw)
+          @tail = tail
+        end
 
-        def initialize(raw = {})
-          @raw = raw
+        def parent?
+          @raw[:type] == 'parent'
         end
 
-        def folders
-          Folders.new(raw[:folders])
+        def folder?
+          @raw[:type] == 'folder'
+        end
+
+        def file?
+          @raw[:type] == 'file'
+        end
+
+        params :id, :node_name, :parent_id, :file_hash, :updated_at
+
+        def as_file
+          {
+            file_name: node_name,
+            file_hash: file_hash,
+            updated_at: updated_at,
+            file_path: file_path,
+            folder_id: parent_id,
+            project_name: @tail.project_name,
+            bucket_name: @tail.bucket_name,
+          }
+        end
+
+        def parent_path
+          return nil if !parent_id
+
+          if !@tail.paths[parent_id]
+            parent = @tail.parents[parent_id]
+            @tail.paths[parent_id] = parent.file_path
+          end
+
+          return @tail.paths[parent_id]
+        end
+
+        def file_path
+          [ parent_path, node_name ].compact.join('/')
         end
       end
 
-      class FilesResponse
-        attr_reader :raw
+      class TailResponse < Etna::Clients::Response
+        attr_reader :project_name, :bucket_name, :paths, :parents
+
+
+        def initialize(project_name, bucket_name, raw = {})
+          super(raw)
+
+          @paths = {}
+
+          @parents = {}
+
+          @project_name = project_name
+          @bucket_name = bucket_name
+
+          @nodes = []
+          raw.each do |raw_node|
+            node = TailNode.new(self, raw_node)
+            if node.parent?
+              @parents[node.id] = node
+            else
+              @nodes.push(node)
+            end
+          end
+        end
+
+        def folders
+          @nodes.filter_map do |node|
+            Folder.new(node.as_folder) if node.folder?
+          end
+        end
+
+        def files
+          @nodes.filter_map do |node|
+            File.new(node.as_file) if node.file?
+          end
+        end
+      end
 
-        def initialize(raw = {})
-          @raw = raw
+      class FoldersResponse < Etna::Clients::Response
+        def folders
+          Folders.new(raw[:folders])
         end
+      end
 
+      class FilesResponse < Etna::Clients::Response
         def files
           Files.new(raw[:files])
         end
@@ -253,37 +342,19 @@ module Etna
         end
       end
 
-      class Files
-        attr_reader :raw
-
-        def initialize(raw = {})
-          @raw = raw
-        end
-
+      class Files < Etna::Clients::Response
         def all
           raw.map { |file| File.new(file) }
         end
       end
 
-      class Folders
-        attr_reader :raw
-
-        def initialize(raw = {})
-          @raw = raw
-        end
-
+      class Folders < Etna::Clients::Response
         def all
           raw.map { |folder| Folder.new(folder) }
         end
       end
 
-      class File
-        attr_reader :raw
-
-        def initialize(raw = {})
-          @raw = raw
-        end
-
+      class File < Etna::Clients::Response
         def with_containing_folder(folder)
           folder_path = folder.is_a?(Folder) ? folder.folder_path : folder
           File.new({}.update(self.raw).update({
@@ -291,17 +362,7 @@ module Etna
           }))
         end
 
-        def file_path
-          raw[:file_path]
-        end
-
-        def project_name
-          raw[:project_name]
-        end
-
-        def bucket_name
-          raw[:bucket_name]
-        end
+        params :file_path, :project_name, :bucket_name, :file_name, :size, :file_hash, :folder_id
 
         def download_path
           raw[:download_url].nil? ?
@@ -313,65 +374,40 @@ module Etna
           raw[:download_url] || ''
         end
 
-        def file_name
-          raw[:file_name]
-        end
-
         def updated_at
           time = raw[:updated_at]
           time.nil? ? nil : Time.parse(time)
         end
 
-        def size
-          raw[:size]
+        def as_magma_file_attribute
+          {
+            path: as_metis_url,
+            original_filename: file_name || ::File.basename(file_path)
+          }
         end
 
-        def file_hash
-          raw[:file_hash]
-        end
-
-        def folder_id
-          raw[:folder_id]
+        def as_metis_url
+          "metis://#{project_name}/#{bucket_name}/#{file_path}"
         end
       end
 
-      class Folder
-        attr_reader :raw
-
-        def initialize(raw = {})
-          @raw = raw
-        end
-
-        def folder_path
-          raw[:folder_path]
-        end
-
-        def folder_name
-          raw[:folder_name]
-        end
-
-        def bucket_name
-          raw[:bucket_name]
-        end
-
-        def project_name
-          raw[:project_name]
-        end
+      class Folder < Etna::Clients::Response
+        params :folder_path, :folder_name, :bucket_name, :project_name, :id
 
         def updated_at
           time = raw[:updated_at]
           time.nil? ? nil : Time.parse(time)
         end
-
-        def id
-          raw[:id]
-        end
       end
 
       class AuthorizeUploadRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)
         include JsonSerializableStruct
       end
 
+      class AuthorizeDownloadRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)
+        include JsonSerializableStruct
+      end
+
       class UploadStartRequest < Struct.new(:file_size, :action, :metis_uid, :next_blob_size, :upload_path, :next_blob_hash, :reset, keyword_init: true)
         include JsonSerializableStruct
 
@@ -392,12 +428,13 @@ module Etna
         end
       end
 
-      class UploadResponse
-        attr_reader :raw
-        def initialize(raw = {})
-          @raw = raw
+      class DownloadResponse < Etna::Clients::Response
+        def download_url
+          raw['download_url']
         end
+      end
 
+      class UploadResponse < Etna::Clients::Response
         def current_byte_position
           raw['current_byte_position'].to_i
         end
@@ -419,6 +456,22 @@ module Etna
         START = UploadAction.new("start")
         BLOB = UploadAction.new("blob")
       end
+
+      class GetFileCountByProjectRequest < Struct.new(:project_names, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+      end
+
+      class GetByteCountByProjectRequest < Struct.new(:project_names, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+      end
     end
   end
-end
+end

data/lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb

@@ -9,24 +9,34 @@ module Etna
         # Since we are doing manual triage of files,
         #   do not automatically copy directory trees.
         # srcs must be a list of full paths to files.
-        def copy_files(srcs, &block)
-          srcs.each do |src|
+        def copy_files(files, &block)
+          files.each do |file|
+            if file.is_a?(Array)
+              src, dest = file
+            else
+              src = dest = file
+            end
+
             if !ingest_filesystem.exist?(src)
               logger&.warn("#{src} does not exist on source filesystem. Skipping.")
+              yield src, false if block_given?
               next
             end
 
             logger&.info("Copying file #{src} (#{Etna::Formatting.as_size(ingest_filesystem.stat(src).size)})")
 
-            # For ingestion triage, just copy over the exact path + filename.
-            copy_file(dest: src, src: src, &block)
+            begin
+              copy_file(dest: dest, src: src, &block)
+            rescue Exception => e
+              yield src, false if block_given?
+            end
           end
         end
 
         def copy_file(dest:, src:, &block)
           ingest_filesystem.with_readable(src, "r") do |io|
             metis_filesystem.do_streaming_upload(io, dest, ingest_filesystem.stat(src).size)
-            yield src if block_given?
+            yield src, true if block_given?
           end
         end
       end

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -197,6 +197,7 @@ module Etna
                 end
               end
             end
+            @last_bytes
           end
         end
       end

data/lib/etna/clients/models.rb

@@ -0,0 +1,19 @@
+module Etna
+  module Clients
+    class Response
+      attr_reader :raw
+
+      def self.params(*params)
+        params.each do |param|
+          self.define_method param do
+            @raw[param]
+          end
+        end
+      end
+
+      def initialize(raw = {})
+        @raw = raw || {}
+      end
+    end
+  end
+end

data/lib/etna/clients/polyphemus/client.rb

@@ -33,6 +33,102 @@ module Etna
             yield res
         end
       end
+
+      def get_config(project_name, config_id, version_number)
+        json = nil
+        @etna_client.post("/api/workflows/#{project_name}/configs/#{config_id}", version_number: version_number) do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def get_runtime_config(project_name, config_id)
+        json = nil
+        @etna_client.get("/api/workflows/#{project_name}/runtime_configs/#{config_id}") do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def update_run(project_name, run_id, updates)
+        payload = {
+          run_id: run_id,
+          workflow_name: updates[:workflow_name],
+          name: updates[:name],
+          config_id: updates[:config_id], 
+          version_number: updates[:version_number],
+          state: updates[:state],
+          orchestrator_metadata: updates[:orchestrator_metadata],
+          output: updates[:output],
+          append_output: updates[:append_output]
+        }.compact
+
+        json = nil
+        @etna_client.post("/api/workflows/#{project_name}/run/update/#{run_id}", payload) do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def get_run(project_name, run_id)
+        json = nil
+        @etna_client.get("/api/workflows/#{project_name}/run/#{run_id}") do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def get_previous_state(project_name, config_id, state: [], collect: false)
+        json = nil
+        @etna_client.post("/api/workflows/#{project_name}/run/previous/#{config_id}",
+          state: state,
+          collect: collect
+        ) do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def update_runtime_config(project_name, config_id, updates)
+        payload = {
+          config_id: updates[:config_id],
+          runtime_config: updates[:runtime_config],
+          run_interval: updates[:run_interval],
+          disabled: updates[:disabled]
+        }.compact          
+
+        json = nil
+        @etna_client.post("/api/workflows/#{project_name}/runtime_config/#{config_id}", payload) do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def get_run_metadata(project_name, config_id)
+        json = nil
+        @etna_client.get("/api/workflows/#{project_name}/runtime_config/#{config_id}") do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def log(project_name:, user:, event:, message:, payload:, signatory:, consolidate:, application:nil)
+        params = {
+          project_name: project_name,
+          user: user,
+          event: event,
+          message: message,
+          payload: payload,
+          consolidate: consolidate,
+          application: application,
+          signatory: signatory
+        }
+        json = nil
+        @etna_client.log_write(params) do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
     end
   end
 end

data/lib/etna/clients.rb

@@ -1,4 +1,6 @@
+require_relative 'clients/models'
 require_relative 'clients/magma'
+require_relative 'clients/gnomon'
 require_relative 'clients/metis'
 require_relative 'clients/janus'
 require_relative 'clients/polyphemus'

data/lib/etna/controller.rb

@@ -16,10 +16,25 @@ module Etna
       @hmac = @request.env['etna.hmac']
     end
 
+    def application
+      Etna::Application.instance
+    end
+
     def log(line)
       @logger.warn(request_msg(line))
     end
 
+    def event_log(params)
+      begin
+        Etna::Application.instance.event_log(**{
+          project_name: @params[:project_name],
+          user: @user
+        }.compact.merge(params))
+      rescue Exception => e
+        log("event_log failed with #{e.backtrace} #{e.message}")
+      end
+    end
+
     def handle_error(e)
       case e
       when Etna::Error
@@ -72,7 +87,16 @@ module Etna
         @response.close
       else
         @response['Content-Type'] = content_type
-        block.call(@response)
+        response = @response
+        # Rack::Response is not a full IO object in Rack 3. JSON.dump calls
+        # flush after writing, so adapt the non-hijack/test response path to
+        # the same small stream interface provided by a hijacked socket.
+        stream = Object.new
+        stream.define_singleton_method(:write) { |data| response.write(data) }
+        stream.define_singleton_method(:<<) { |data| write(data) }
+        stream.define_singleton_method(:flush) {}
+
+        block.call(stream)
         @response.finish
       end
     end
@@ -86,7 +110,7 @@ Subject: #{subject}
 #{content}
 MESSAGE_END
 
-      unless @server.send(:application).test?
+      unless application.test?
         Net::SMTP.start('smtp.ucsf.edu') do |smtp|
           smtp.send_message message, 'noreply@janus', to_email
         end
@@ -108,7 +132,7 @@ MESSAGE_END
     def route_url(name, params={})
       path = route_path(name,params)
       return nil unless path
-      @request.scheme + '://' + @request.host + path
+      @request.scheme + '://' + application.host + path
     end
 
     # methods for returning a view
@@ -133,8 +157,8 @@ MESSAGE_END
     end
 
     def config_hosts
-      [:janus, :magma, :timur, :metis, :vulcan, :polyphemus, :gnomon].map do |host|
-        [ :"#{host}_host", @server.send(:application).config(host)&.dig(:host) ]
+      [:janus, :magma, :timur, :metis, :vulcan, :polyphemus, :gnomon, :vesta].map do |host|
+        [ :"#{host}_host", application.config(host)&.dig(:host) ]
       end.to_h.compact
     end
 
@@ -162,8 +186,9 @@ MESSAGE_END
       self.class.name.sub("Kernel::", "").sub("Controller", "").downcase
     end
 
-    def success(msg, content_type='text/plain')
+    def success(msg, content_type='text/plain', disposition='not given')
       @response['Content-Type'] = content_type
+      @response['Content-Disposition'] = disposition unless disposition=='not given'
       @response.write(msg)
       @response.finish
     end

data/lib/etna/cross_origin.rb

@@ -27,7 +27,7 @@ module Etna
     end
 
     def origin_allowed?(request)
-      subdomain(URI.parse(header(request, :origin)).host) == subdomain(request.host)
+      subdomain(URI.parse(header(request, :origin)).host) == subdomain(Etna::Application.instance.host)
     end
 
     def actual_response(request)

data/lib/etna/cwl.rb

@@ -240,7 +240,7 @@ module Etna
       end
 
       PRIMITIVE_TYPE = EnumLoader.new("null", "boolean", "int", "long", "float", "double", "string")
-      NOMINAL_TYPE = EnumLoader.new("File")
+      NOMINAL_TYPE = EnumLoader.new("File", "MetisFile", "MetisFolder", "MetisPath", "MetisCSVorTSV")
     end
 
     class OptionalLoader < Loader

data/lib/etna/errors.rb

@@ -40,4 +40,12 @@ module Etna
       @level = Logger::ERROR
     end
   end
+
+  class TooManyRequests < Etna::Error
+    def initialize(msg = 'Too many requests', status = 429)
+      super
+      @level = Logger::ERROR
+    end
+  end
+
 end