etna 0.1.52 → 0.2.0

data/lib/etna/clients/polyphemus/client.rb

@@ -33,6 +33,102 @@ module Etna
             yield res
         end
       end
+
+      def get_config(project_name, config_id, version_number)
+        json = nil
+        @etna_client.post("/api/workflows/#{project_name}/configs/#{config_id}", version_number: version_number) do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def get_runtime_config(project_name, config_id)
+        json = nil
+        @etna_client.get("/api/workflows/#{project_name}/runtime_configs/#{config_id}") do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def update_run(project_name, run_id, updates)
+        payload = {
+          run_id: run_id,
+          workflow_name: updates[:workflow_name],
+          name: updates[:name],
+          config_id: updates[:config_id], 
+          version_number: updates[:version_number],
+          state: updates[:state],
+          orchestrator_metadata: updates[:orchestrator_metadata],
+          output: updates[:output],
+          append_output: updates[:append_output]
+        }.compact
+
+        json = nil
+        @etna_client.post("/api/workflows/#{project_name}/run/update/#{run_id}", payload) do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def get_run(project_name, run_id)
+        json = nil
+        @etna_client.get("/api/workflows/#{project_name}/run/#{run_id}") do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def get_previous_state(project_name, config_id, state: [], collect: false)
+        json = nil
+        @etna_client.post("/api/workflows/#{project_name}/run/previous/#{config_id}",
+          state: state,
+          collect: collect
+        ) do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def update_runtime_config(project_name, config_id, updates)
+        payload = {
+          config_id: updates[:config_id],
+          runtime_config: updates[:runtime_config],
+          run_interval: updates[:run_interval],
+          disabled: updates[:disabled]
+        }.compact          
+
+        json = nil
+        @etna_client.post("/api/workflows/#{project_name}/runtime_config/#{config_id}", payload) do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def get_run_metadata(project_name, config_id)
+        json = nil
+        @etna_client.get("/api/workflows/#{project_name}/runtime_config/#{config_id}") do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
+
+      def log(project_name:, user:, event:, message:, payload:, signatory:, consolidate:, application:nil)
+        params = {
+          project_name: project_name,
+          user: user,
+          event: event,
+          message: message,
+          payload: payload,
+          consolidate: consolidate,
+          application: application,
+          signatory: signatory
+        }
+        json = nil
+        @etna_client.log_write(params) do |res|
+          json = JSON.parse(res.body)
+        end
+        json
+      end
     end
   end
 end

data/lib/etna/clients.rb

@@ -1,4 +1,6 @@
+require_relative 'clients/models'
 require_relative 'clients/magma'
+require_relative 'clients/gnomon'
 require_relative 'clients/metis'
 require_relative 'clients/janus'
 require_relative 'clients/polyphemus'

data/lib/etna/controller.rb

@@ -16,10 +16,25 @@ module Etna
       @hmac = @request.env['etna.hmac']
     end
 
+    def application
+      Etna::Application.instance
+    end
+
     def log(line)
       @logger.warn(request_msg(line))
     end
 
+    def event_log(params)
+      begin
+        Etna::Application.instance.event_log(**{
+          project_name: @params[:project_name],
+          user: @user
+        }.compact.merge(params))
+      rescue Exception => e
+        log("event_log failed with #{e.backtrace} #{e.message}")
+      end
+    end
+
     def handle_error(e)
       case e
       when Etna::Error
@@ -72,7 +87,16 @@ module Etna
         @response.close
       else
         @response['Content-Type'] = content_type
-        block.call(@response)
+        response = @response
+        # Rack::Response is not a full IO object in Rack 3. JSON.dump calls
+        # flush after writing, so adapt the non-hijack/test response path to
+        # the same small stream interface provided by a hijacked socket.
+        stream = Object.new
+        stream.define_singleton_method(:write) { |data| response.write(data) }
+        stream.define_singleton_method(:<<) { |data| write(data) }
+        stream.define_singleton_method(:flush) {}
+
+        block.call(stream)
         @response.finish
       end
     end
@@ -86,7 +110,7 @@ Subject: #{subject}
 #{content}
 MESSAGE_END
 
-      unless @server.send(:application).test?
+      unless application.test?
         Net::SMTP.start('smtp.ucsf.edu') do |smtp|
           smtp.send_message message, 'noreply@janus', to_email
         end
@@ -108,7 +132,7 @@ MESSAGE_END
     def route_url(name, params={})
       path = route_path(name,params)
       return nil unless path
-      @request.scheme + '://' + @request.host + path
+      @request.scheme + '://' + application.host + path
     end
 
     # methods for returning a view
@@ -133,8 +157,8 @@ MESSAGE_END
     end
 
     def config_hosts
-      [:janus, :magma, :timur, :metis, :vulcan, :polyphemus, :gnomon].map do |host|
-        [ :"#{host}_host", @server.send(:application).config(host)&.dig(:host) ]
+      [:janus, :magma, :timur, :metis, :vulcan, :polyphemus, :gnomon, :vesta].map do |host|
+        [ :"#{host}_host", application.config(host)&.dig(:host) ]
       end.to_h.compact
     end
 
@@ -162,8 +186,9 @@ MESSAGE_END
       self.class.name.sub("Kernel::", "").sub("Controller", "").downcase
     end
 
-    def success(msg, content_type='text/plain')
+    def success(msg, content_type='text/plain', disposition='not given')
       @response['Content-Type'] = content_type
+      @response['Content-Disposition'] = disposition unless disposition=='not given'
       @response.write(msg)
       @response.finish
     end

data/lib/etna/cross_origin.rb

@@ -27,7 +27,7 @@ module Etna
     end
 
     def origin_allowed?(request)
-      subdomain(URI.parse(header(request, :origin)).host) == subdomain(request.host)
+      subdomain(URI.parse(header(request, :origin)).host) == subdomain(Etna::Application.instance.host)
     end
 
     def actual_response(request)

data/lib/etna/cwl.rb

@@ -240,7 +240,7 @@ module Etna
       end
 
       PRIMITIVE_TYPE = EnumLoader.new("null", "boolean", "int", "long", "float", "double", "string")
-      NOMINAL_TYPE = EnumLoader.new("File")
+      NOMINAL_TYPE = EnumLoader.new("File", "MetisFile", "MetisFolder", "MetisPath", "MetisCSVorTSV")
     end
 
     class OptionalLoader < Loader

data/lib/etna/errors.rb

@@ -40,4 +40,12 @@ module Etna
       @level = Logger::ERROR
     end
   end
+
+  class TooManyRequests < Etna::Error
+    def initialize(msg = 'Too many requests', status = 429)
+      super
+      @level = Logger::ERROR
+    end
+  end
+
 end

data/lib/etna/filesystem.rb

@@ -70,7 +70,6 @@ module Etna
         rd, wd = IO.pipe
 
         cmd = mkcommand(rd, wd, file, opts, size_hint: size_hint)
-        puts "in mkio: #{cmd}"
         pid = spawn(*cmd)
         q = Queue.new
 
@@ -103,173 +102,6 @@ module Etna
       end
     end
 
-    class AsperaCliFilesystem < Filesystem
-      include WithPipeConsumer
-
-      def initialize(ascli_bin:, ascp_bin:, host:, username:, password: nil, key_file: nil, port: 33001)
-        @ascli_bin = ascli_bin
-        @ascp_bin = ascp_bin
-        @username = username
-        @password = password
-        @key_file = key_file
-        @host = host
-        @port = port
-
-        @config_file = File.join(Dir.mktmpdir, "config.yml")
-        config = {}
-        config["config"] = {"version" => `#{ascli_bin} --version`.chomp}
-        config["default"] = {"server" => "clifilesystem"}
-        server_config = config["clifilesystem"] = {
-            "url" => "ssh://#{host}:#{port}",
-            "username" => username,
-            "ssh_options" => {append_all_supported_algorithms: true},
-        }
-
-        if password
-          server_config["password"] = password
-        elsif key_file
-          server_config["ssh_keys"] = key_file
-        else
-          raise "One of password or key_file must be provided"
-        end
-
-        ::File.open(@config_file, "w") do |file|
-          file.write(config.to_yaml)
-        end
-      end
-
-      def run_ascli_cmd(cmd, *opts)
-        output, status = Open3.capture2(@ascli_bin, "server", cmd, *opts, "--format=json", "--config=#{@config_file}")
-
-        if status.success?
-          return JSON.parse(output)
-        end
-
-        nil
-      end
-
-      def with_writeable(dest, opts = 'w', size_hint: nil, &block)
-        mkio(dest, opts, size_hint: size_hint, &block)
-      end
-
-      def with_readable(src, opts = 'r', &block)
-        mkio(src, opts, &block)
-      end
-
-      def mkdir_p(dir)
-        raise "Failed to mkdir #{dir}" unless run_ascli_cmd("mkdir", dir)
-      end
-
-      def rm_rf(dir)
-        raise "Failed to rm_rf #{dir}" unless run_ascli_cmd("rm", dir)
-      end
-
-      def tmpdir
-        tmpdir = "/Upload/Temp/#{SecureRandom.hex}"
-        mkdir_p(tmpdir)
-        tmpdir
-      end
-
-      def exist?(src)
-        !run_ascli_cmd("ls", src).nil?
-      end
-
-      def mv(src, dest)
-        raise "Failed to mv #{src} to #{dest}" unless run_ascli_cmd("mv", src, dest)
-      end
-
-      def mkcommand(rd, wd, file, opts, size_hint: nil)
-        env = {}
-        cmd = [env, @ascp_bin]
-
-        if @password
-          env['ASPERA_SCP_PASS'] = @password
-        else
-          cmd << "-i"
-          cmd << @key_file
-        end
-
-        cmd << "-O"
-        cmd << @port.to_s
-
-        cmd << "-P"
-        cmd << @port.to_s
-
-        cmd << "-v"
-        cmd << "-k"
-        cmd << "0"
-        cmd << "--file-manifest=text"
-        cmd << "--file-manifest-path=/var/tmp"
-        cmd << "--file-checksum=md5"
-        cmd << "-l"
-        cmd << "100m"
-        cmd << "--overwrite=always"
-
-        remote_path = file
-        # https://download.asperasoft.com/download/docs/entsrv/3.9.1/es_admin_linux/webhelp/index.html#dita/stdio_2.html
-        local_path = "stdio://"
-        if size_hint
-          local_path += "/?#{size_hint}"
-        end
-
-        if opts.include?('r')
-          cmd << '--mode=recv'
-          cmd << "--host=#{@host}"
-          cmd << "--user=#{@username}"
-          cmd << remote_path
-          cmd << local_path
-
-          cmd << {out: wd}
-        elsif opts.include?('w')
-          cmd << '--mode=send'
-          cmd << "--host=#{@host}"
-          cmd << "--user=#{@username}"
-          cmd << local_path
-          cmd << remote_path
-
-          cmd << {in: rd}
-        end
-
-        puts "end of mkcmd: #{cmd}"
-        cmd
-      end
-    end
-
-    # Genentech's aspera deployment doesn't support modern commands, unfortunately...
-    class GeneAsperaCliFilesystem < AsperaCliFilesystem
-      def mkdir_p(dest)
-        # Pass through -- this file system creates containing directories by default, womp womp.
-      end
-
-      def mkcommand(rd, wd, file, opts, size_hint: nil)
-        if opts.include?('w')
-          super.map do |e|
-            if e.instance_of?(String) && e.start_with?("stdio://")
-              "stdio-tar://"
-            elsif e == file
-              dir = ::File.dirname(file)
-              dir[0] == '/' ? dir : "/#{dir}"
-            else
-              e
-            end
-          end.insert(2, "-d")
-        else
-          super
-        end
-      end
-
-      def with_writeable(dest, opts = 'w', size_hint: nil, &block)
-        raise "#{self.class.name} requires size_hint in with_writeable" if size_hint.nil?
-
-        super do |io|
-          io.write("File: #{::File.basename(dest)}\n")
-          io.write("Size: #{size_hint}\n")
-          io.write("\n")
-          yield io
-        end
-      end
-    end
-
     class Metis < Filesystem
       attr_reader :project_name, :bucket_name
 
@@ -438,9 +270,11 @@ module Etna
       end
 
       def sftp_file_from_path(src)
-        file = ls(::File.dirname(src)).split("\n").map do |listing|
+        files = ls(::File.dirname(src)).split("\n").map do |listing|
           SftpFile.new(listing)
-        end.select do |file|
+        end
+
+        file = files.select do |file|
           file.name == ::File.basename(src)
         end
 
@@ -457,6 +291,7 @@ module Etna
         cmd << authn
         cmd << "-o"
         cmd << "-"
+        cmd << "-s"
         cmd << "-N"
         cmd << url(file)
 

data/lib/etna/hmac.rb

@@ -46,7 +46,7 @@ module Etna
     end
 
     def signature
-      @application.sign.hmac(text_to_sign, @application.config(:hmac_keys)[@id])
+      @application.sign.hmac(text_to_sign, @application.config(@id)[:hmac_key])
     end
 
     private
@@ -60,8 +60,8 @@ module Etna
     end
 
     def valid_id?
-      return false if !@application.config(:hmac_keys)
-      @application.config(:hmac_keys).key?(@id)
+      return false if !@application.config(@id)
+      @application.config(@id).key?(:hmac_key)
     end
 
     # This scheme is adapted from the Hawk spec
@@ -79,7 +79,7 @@ module Etna
         # these are set as headers or params
         @nonce,
         @id,
-        @headers.to_json,
+        JSON.generate(@headers),
         @expiration,
       ].join("\n")
     end

data/lib/etna/instrumentation.rb

@@ -6,8 +6,8 @@ module Etna
           orig_method_name = :"#{method_name}_without_time_it"
           self.alias_method orig_method_name, method_name
 
-          self.define_method method_name do |*args|
-            time_it(orig_method_name, args, &metric_block)
+          self.define_method method_name do |*args, **kwargs|
+            time_it(orig_method_name, args, kwargs, &metric_block)
           end
         end
       end
@@ -20,11 +20,11 @@ module Etna
       end
     end
 
-    def time_it(method_name, args, &metric_block)
+    def time_it(method_name, args, kwargs = {}, &metric_block)
       if has_yabeda?
         start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         begin
-          return send(method_name, *args)
+          return send(method_name, *args, **kwargs)
         ensure
           dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
           if block_given?
@@ -37,13 +37,13 @@ module Etna
           metric.measure(tags, dur)
         end
       else
-        return send(method_name, *args, **kwds)
+        return send(method_name, *args, **kwargs)
       end
     end
 
     def with_yabeda_tags(tags, &block)
       if has_yabeda?
-        Yabeda.with_tags(tags, &block)
+        Yabeda.with_tags(**tags, &block)
       else
         yield
       end

data/lib/etna/redirect.rb

@@ -26,7 +26,7 @@ module Etna
     private
 
     def matches_domain?(path)
-      top_domain(@request.hostname) == top_domain(URI(path).host)
+      top_domain(Etna::Application.instance.host) == top_domain(URI(path).host)
     end
 
     def top_domain(host_name)

data/lib/etna/remote.rb

@@ -1,4 +1,6 @@
 require "net/ssh"
+require "net/scp"
+require "tempfile"
 
 module Etna
   class RemoteSSH
@@ -11,6 +13,7 @@ module Etna
       @host = host
       @port = port
       @root = root
+      raise "RemoteSSH must have host, username and password" unless @host && @username && @password
     end
 
     def ssh
@@ -23,8 +26,8 @@ module Etna
       raise RemoteSSHError.new("Unable to mkdir -p, #{output}") unless 0 == output.exitstatus
     end
 
-    def lftp_get(username:, password:, host:, remote_filename:, &block)
-      full_local_path = ::File.join(@root, host, remote_filename)
+    def lftp_get(username:, password:, host:, remote_filename:, local_filename:, &block)
+      full_local_path = local_filename
       full_local_dir = ::File.dirname(full_local_path)
       mkdir_p(full_local_dir)
 
@@ -34,5 +37,18 @@ module Etna
       raise RemoteSSHError.new("LFTP get failure: #{output}") unless 0 == output.exitstatus
       yield remote_filename if block_given?
     end
+
+    def file_upload(remote_path, content)
+      begin
+        Tempfile.create do |temp_file|
+          temp_file.binmode
+          temp_file.write(content)
+          temp_file.flush
+          ssh.scp.upload!(temp_file.path, remote_path)
+        end
+      rescue StandardError => e
+        raise RemoteSSHError.new("File upload failed: #{e.message}")
+      end
+    end
   end
 end

data/lib/etna/route.rb

@@ -221,6 +221,9 @@ module Etna
       #   for the project
       return false if user.permissions.keys.include?(params[:project_name])
 
+      # Only check for a CC if the user is not flagged 'external'
+      return false if user.has_flag?('external')
+
       !janus.community_projects(user.token).select do |project|
         project.project_name == params[:project_name]
       end.first.nil?

data/lib/etna/spec/event_log.rb

@@ -0,0 +1,16 @@
+require 'webmock/rspec'
+
+def stub_event_log(project_name='labors')
+  polyphemus_host = Etna::Application.instance.config(:polyphemus)[:host]
+  WebMock.stub_request(:post, /#{polyphemus_host}\/api\/log\/#{project_name}/).
+    to_return(status: 200, body: { log: 1 }.to_json, headers: {'Content-Type': 'application/json'})
+
+  WebMock.stub_request(:options, polyphemus_host).
+    to_return({
+    status: 200,
+    headers: { 'Content-Type': "application/json" },
+    body: [
+      { :method => "POST", :route => "/api/log/:project_name/write", :name => "log_write", :params => ["project_name"] }
+    ].to_json
+  })
+end

data/lib/etna/spec/vcr.rb

@@ -71,8 +71,8 @@ def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
       end
     end
 
-    if File.exists?('log')
-      c.debug_logger = File.open('log/vcr_debug.log', 'w')
+    if ::File.exist?('log')
+      c.debug_logger = ::File.open('log/vcr_debug.log', 'w')
     end
 
     c.default_cassette_options = {
@@ -147,4 +147,4 @@ def prepare_vcr_secret
   digest = Digest::SHA256.new
   digest.update(secret)
   digest.digest
-end
+end

data/lib/etna/test_auth.rb

@@ -80,7 +80,7 @@ module Etna
 
       hmac_params = {
         method: request.request_method,
-        host: request.host,
+        host: application.host,
         path: request.path,
 
         expiration: etna_param(request, :expiration) || DateTime.now.iso8601,

data/lib/etna/user.rb

@@ -68,10 +68,6 @@ module Etna
       is_supereditor? || has_any_role?(project, :admin)
     end
 
-    def active? project=nil
-      permissions.keys.length > 0
-    end
-
     def display_name
       [ email, name ].join('|')
     end

data/lib/helpers.rb

@@ -51,6 +51,14 @@ module WithEtnaClients
         **EtnaApp.instance.config(:magma, environment) || {})
   end
 
+  def gnomon_client(logger: nil)
+    @gnomon_client ||= Etna::Clients::Gnomon.new(
+        token: token,
+        ignore_ssl: EtnaApp.instance.config(:ignore_ssl),
+        logger: logger,
+        **EtnaApp.instance.config(:magma, environment) || {})
+  end
+
   def metis_client(logger: nil)
     @metis_client ||= Etna::Clients::Metis.new(
         token: token,

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.52
+  version: 0.2.0
 platform: ruby
 authors:
 - Saurabh Asthana
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-06-30 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -157,6 +156,9 @@ files:
 - lib/etna/clients.rb
 - lib/etna/clients/base_client.rb
 - lib/etna/clients/enum.rb
+- lib/etna/clients/gnomon.rb
+- lib/etna/clients/gnomon/client.rb
+- lib/etna/clients/gnomon/models.rb
 - lib/etna/clients/janus.rb
 - lib/etna/clients/janus/client.rb
 - lib/etna/clients/janus/models.rb
@@ -193,6 +195,7 @@ files:
 - lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb
 - lib/etna/clients/metis/workflows/walk_metis_diff_workflow.rb
 - lib/etna/clients/metis/workflows/walk_metis_workflow.rb
+- lib/etna/clients/models.rb
 - lib/etna/clients/polyphemus.rb
 - lib/etna/clients/polyphemus/client.rb
 - lib/etna/clients/polyphemus/models.rb
@@ -229,6 +232,7 @@ files:
 - lib/etna/sign_service.rb
 - lib/etna/spec.rb
 - lib/etna/spec/auth.rb
+- lib/etna/spec/event_log.rb
 - lib/etna/spec/vcr.rb
 - lib/etna/symbolize_params.rb
 - lib/etna/synchronize_db.rb
@@ -255,8 +259,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Base classes for Mount Etna applications
 test_files: []