etna 0.1.46 → 0.1.47

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf5d46399bfc782d537d246013aeb1cc16bf3b56c8040f84baa92a36c5949fda
-  data.tar.gz: 32fabdd9b634b7a0026b8277a55635c16be52e30a60cd7568e8c68938d44d221
+  metadata.gz: eb695213d2c20c23465eb992a31fa8b0d30c80d78f0c83f672fbb45317ff33d4
+  data.tar.gz: efd86410cf85007a8bd1c957882a44ab5a0334fd0b54b87a8a140ff4203b74c5
 SHA512:
-  metadata.gz: f7b48496c8b36afe4aa6f08ef911913bfee6233b3ce50f17e723b38aef03f3aa2b7c9ee6beab94748fa42ff67bfcf68bb73145962ce102d79593055025c439d4
-  data.tar.gz: 4f074e556b455f2e6a2e36b30212b781316f198acfa99879059c803d2254ae701bd784adf515e1fb3427734a299546d42a182f08908645ff340ae5a9581dcb91
+  metadata.gz: c288d8ecc5d518523a9ae7591c1edd8d85030ea84fe0b9f0f468d1f8610b27b7b3d4f4ca3fe28b0cf0f49c77baeedf168707193b95d2489729a6ab6bab966044
+  data.tar.gz: e987626f648fee08acd56c5615d63200b5d3813f511a8043d6afa9987f0bdad4fd72e485d25982702ebb748a00f0a7703372fdc45cf962550e171472b17e7ab8

data/lib/commands.rb

@@ -206,7 +206,7 @@ class EtnaApp
     end
   end
 
-  class Administrate
+  class Janus
     include Etna::CommandExecutor
 
     class Token
@@ -238,6 +238,59 @@ class EtnaApp
         end
       end
     end
+  end
+
+  class Magma
+    include Etna::CommandExecutor
+
+    class Materialize < Etna::Command
+      include WithEtnaClients
+
+      string_flags << "--project-name"
+      string_flags << "--log-file"
+      string_flags << "--log-level"
+      string_flags << "--concurrency"
+
+
+      def execute(project_name:, log_file:'/dev/stdout', log_level: ::Logger::INFO, concurrency: 1)
+        logger = Etna::Logger.new(log_file, 0, 1048576)
+
+        logger.level = log_level
+
+        workflow = Etna::Clients::Magma::MaterializeDataWorkflow.new(
+            model_attributes_mask: model_attribute_pairs(project_name),
+            record_names: 'all',
+            model_filters: {},
+            metis_client: metis_client,
+            magma_client: magma_client,
+            logger: logger,
+            project_name: project_name,
+            model_name: 'project', filesystem: filesystem,
+            concurrency: concurrency.to_i)
+
+        workflow.materialize_all(project_name)
+        logger.info("Done")
+      end
+
+      def model_attribute_pairs(project_name)
+        models = magma_client.retrieve(Etna::Clients::Magma::RetrievalRequest.new(
+            project_name: project_name,
+            model_name: 'all',
+            attribute_names: 'all',
+            record_names: []
+        )).models
+
+        result = models.model_keys.map do |model_name|
+          [ model_name, models.model(model_name).template.attributes.attribute_keys ]
+        end.to_h
+
+        result
+      end
+
+      def filesystem
+        @filesystem ||= Etna::Filesystem.new
+      end
+    end
 
     class Project
       include Etna::CommandExecutor

data/lib/etna/application.rb

@@ -68,38 +68,43 @@ module Etna::Application
     [path.map(&:to_sym), YAML.load(File.read(value))]
   end
 
+  def controller_tags
+    [:controller, :action, :project_name, :user_hash]
+  end
+
   def setup_yabeda
     application = self.id
+    ctags = self.controller_tags
     Yabeda.configure do
       default_tag :application, application
+      default_tag :project_name, 'unknown'
+      default_tag :controller, 'none'
+      default_tag :action, 'none'
+      default_tag :user_hash, 'unknown'
 
       group :etna do
         histogram :response_time do
           comment "Time spent by a controller returning a response"
           unit :seconds
-          tags [:controller, :action, :user_hash, :project_name]
+          tags ctags
+          buckets [0.01, 0.1, 0.3, 0.5, 1, 5]
+        end
+
+        histogram :perf do
+          comment "Time spent inside code path"
+          unit :seconds
+          tags ctags + [:class_name, :method_name]
           buckets [0.01, 0.1, 0.3, 0.5, 1, 5]
         end
 
         counter :visits do
           comment "Counts visits to the controller"
-          tags [:controller, :action, :user_hash, :project_name]
+          tags ctags
         end
 
         counter :rollbar_errors do
           comment "Counts errors detected by and sent to rollbar"
-        end
-
-        gauge :last_command_completion do
-          comment "Unix time of last time command was completed"
-          tags [:command, :status, :application]
-        end
-
-        histogram :command_runtime do
-          comment "Time spent processing a given command"
-          tags [:command, :status, :application]
-          unit :seconds
-          buckets [0.1, 1, 5, 60, 300, 1500]
+          tags ctags
         end
       end
     end
@@ -176,9 +181,6 @@ module Etna::Application
   end
 
   def run_command(config, *args, &block)
-    application = self.id
-    status = 'success'
-    start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     cmd, cmd_args, cmd_kwds = find_command(*args)
 
     begin
@@ -190,18 +192,7 @@ module Etna::Application
       cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
     rescue => e
       Rollbar.error(e)
-      status = 'failed'
       raise
-    ensure
-      if defined?(Yabeda) && Yabeda.configured?
-        tags = { command: cmd.class.name, status: status, application: application }
-        dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
-
-        Yabeda.etna.last_command_completion.set(tags, Time.now.to_i)
-        Yabeda.etna.command_runtime.measure(tags, dur)
-
-        write_job_metrics("run_command.#{cmd.class.name}")
-      end
     end
   end
 end

data/lib/etna/auth.rb

@@ -18,7 +18,7 @@ module Etna
       if [ approve_noauth(request), approve_hmac(request), approve_user(request) ].all?{|approved| !approved}
         return fail_or_redirect(request)
       end
-  
+
       @app.call(request.env)
     end
 
@@ -97,7 +97,7 @@ module Etna
 
       return payload unless route
 
-      begin      
+      begin
         permissions = permissions(payload)
 
         janus.resource_projects(token).each do |resource_project|

data/lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

@@ -41,6 +41,7 @@ module Etna
 
             begin
               if (error = errors.pop(true))
+                logger&.error(error)
                 raise error
               end
             rescue ThreadError
@@ -51,6 +52,7 @@ module Etna
               begin
                 materialize_record(dest, template, document)
               rescue => e
+                logger&.error(e)
                 errors << e
               ensure
                 semaphore.release
@@ -62,6 +64,7 @@ module Etna
 
           begin
             if (error = errors.pop(true))
+              logger&.error(error)
               raise error
             end
           rescue ThreadError
@@ -105,6 +108,7 @@ module Etna
             end
 
             dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: "_#{attr_name}_#{idx}#{File.extname(filename)}"))
+            filesystem.mkdir_p(File.dirname(dest_file))
             sync_metis_data_workflow.copy_file(dest: dest_file, url: url, stub: stub_files)
             record_to_serialize[attr_name] << {file: dest_file, original_filename: filename}
           end

data/lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb

@@ -22,36 +22,56 @@ module Etna
         end
 
         def copy_file(dest:, url:, stub: false)
-          # This does not work due to the magma bucket's restrictions, but if it did work, it'd be super sweet.
-          # url_match = DOWNLOAD_REGEX.match(url)
-          #
-          # if filesystem.instance_of?(Etna::Filesystem::Metis) && !url_match.nil?
-          #   bucket_name = url_match[:bucket_name]
-          #   project_name = url_match[:project_name]
-          #   file_path = url_match[:file_path]
-          #
-          #   metis_client.copy_files(
-          #     Etna::Clients::Metis::CopyFilesRequest.new(
-          #       project_name: project_name,
-          #       revisions: [
-          #         Etna::Clients::Metis::CopyRevision.new(
-          #           source: "metis://#{project_name}/#{bucket_name}/#{file_path}",
-          #           dest: "metis://#{filesystem.project_name}/#{filesystem.bucket_name}#{dest}",
-          #         )
-          #       ]
-          #     )
-          #   )
-          # end
+          url_match = DOWNLOAD_REGEX.match(url)
+
+          if filesystem.instance_of?(Etna::Filesystem::Metis) && !url_match.nil?
+            bucket_name = url_match[:bucket_name]
+            project_name = url_match[:project_name]
+            file_path = url_match[:file_path]
+
+            # ensure target parent directory exists
+            metis_client.ensure_parent_folder_exists(
+              project_name: filesystem.project_name,
+              bucket_name: filesystem.bucket_name,
+              path: dest
+            )
+
+            metis_client.copy_files(
+              Etna::Clients::Metis::CopyFilesRequest.new(
+                project_name: project_name,
+                revisions: [
+                  Etna::Clients::Metis::CopyRevision.new(
+                    source: "metis://#{project_name}/#{bucket_name}/#{file_path}",
+                    dest: "metis://#{filesystem.project_name}/#{filesystem.bucket_name}/#{dest}",
+                  )
+                ]
+              )
+            )
+
+            return
+          end
 
           metadata = metis_client.file_metadata(url)
           size = metadata[:size]
 
+          begin
+            if filesystem.exist?(dest) && filesystem.stat(dest).size == size
+              logger&.info "Already downloaded #{dest}"
+              return
+            end
+          rescue Etna::Filesystem::Error => e
+            unless e.message =~ /stat not supported/
+              raise e
+            end
+          end
+
           tmp_file = dest
           upload_timings = []
           upload_amount = 0
           last_rate = 0.00001
           remaining = size
 
+          logger&.info "Downloading #{dest} - #{Etna::Formatting.as_size(size)}"
           filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
             if stub
               io.write("(stub) #{size} bytes")
@@ -78,7 +98,7 @@ module Etna
                 rate = upload_amount / (end_time - start_time)
 
                 if rate / last_rate > 1.3 || rate / last_rate < 0.7
-                  logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second, #{Etna::Formatting.as_size(remaining)} remaining")
+                  logger&.debug("Uploading #{Etna::Formatting.as_size(rate)} per second, #{Etna::Formatting.as_size(remaining)} remaining")
 
                   if rate == 0
                     last_rate = 0.0001
@@ -87,7 +107,6 @@ module Etna
                   end
                 end
               end
-
             end
           end
         end

data/lib/etna/command.rb

@@ -136,7 +136,7 @@ module Etna
             "<#{name}>..."
           when :keyrest
             "[flags...]"
-          when :key
+          when :key, :keyreq
             flag = "--#{name.to_s.gsub('_', '-')}"
             if self.class.boolean_flags.include?(flag)
               "[#{flag}]"

data/lib/etna/controller.rb

@@ -133,7 +133,7 @@ MESSAGE_END
     end
 
     def config_hosts
-      [:janus, :magma, :timur, :metis, :vulcan, :polyphemus].map do |host|
+      [:janus, :magma, :timur, :metis, :vulcan, :polyphemus, :gnomon].map do |host|
         [ :"#{host}_host", @server.send(:application).config(host)&.dig(:host) ]
       end.to_h.compact
     end

data/lib/etna/filesystem.rb

@@ -14,45 +14,48 @@ module Etna
       ::File.open(dest, opts, &block)
     end
 
+    class Error < StandardError
+    end
+
     def ls(dir)
-      raise "ls not supported by #{self.class.name}" unless self.class == Filesystem
+      raise Etna::Filesystem::Error, "ls not supported by #{self.class.name}" unless self.class == Filesystem
       ::Dir.entries(dir).select { |p| !p.start_with?('.') }.map do |path|
         ::File.file?(::File.join(dir, path)) ? [:file, path] : [:dir, path]
       end
     end
 
     def with_readable(src, opts = 'r', &block)
-      raise "with_readable not supported by #{self.class.name}" unless self.class == Filesystem
+      raise Etna::Filesystem::Error, "with_readable not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.open(src, opts, &block)
     end
 
     def mkdir_p(dir)
-      raise "mkdir_p not supported by #{self.class.name}" unless self.class == Filesystem
+      raise Etna::Filesystem::Error, "mkdir_p not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.mkdir_p(dir)
     end
 
     def rm_rf(dir)
-      raise "rm_rf not supported by #{self.class.name}" unless self.class == Filesystem
+      raise Etna::Filesystem::Error, "rm_rf not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.rm_rf(dir)
     end
 
     def tmpdir
-      raise "tmpdir not supported by #{self.class.name}" unless self.class == Filesystem
+      raise Etna::Filesystem::Error, "tmpdir not supported by #{self.class.name}" unless self.class == Filesystem
       ::Dir.mktmpdir
     end
 
     def exist?(src)
-      raise "exist? not supported by #{self.class.name}" unless self.class == Filesystem
+      raise Etna::Filesystem::Error, "exist? not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.exist?(src)
     end
 
     def mv(src, dest)
-      raise "mv not supported by #{self.class.name}" unless self.class == Filesystem
+      raise Etna::Filesystem::Error, "mv not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.mv(src, dest)
     end
 
     def stat(src)
-      raise "stat not supported by #{self.class.name}" unless self.class == Filesystem
+      raise Etna::Filesystem::Error, "stat not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.stat(src)
     end
 
@@ -66,7 +69,9 @@ module Etna
       def mkio(file, opts, size_hint: nil, &block)
         rd, wd = IO.pipe
 
-        pid = spawn(*mkcommand(rd, wd, file, opts, size_hint: size_hint))
+        cmd = mkcommand(rd, wd, file, opts, size_hint: size_hint)
+        puts "in mkio: #{cmd}"
+        pid = spawn(*cmd)
         q = Queue.new
 
         closer = Thread.new do
@@ -184,9 +189,22 @@ module Etna
           cmd << @key_file
         end
 
+        cmd << "-O"
+        cmd << @port.to_s
+
         cmd << "-P"
         cmd << @port.to_s
 
+        cmd << "-v"
+        cmd << "-k"
+        cmd << "0"
+        cmd << "--file-manifest=text"
+        cmd << "--file-manifest-path=/var/tmp"
+        cmd << "--file-checksum=md5"
+        cmd << "-l"
+        cmd << "100m"
+        cmd << "--overwrite=always"
+
         remote_path = file
         # https://download.asperasoft.com/download/docs/entsrv/3.9.1/es_admin_linux/webhelp/index.html#dita/stdio_2.html
         local_path = "stdio://"
@@ -212,6 +230,7 @@ module Etna
           cmd << {in: rd}
         end
 
+        puts "end of mkcmd: #{cmd}"
         cmd
       end
     end
@@ -228,7 +247,8 @@ module Etna
             if e.instance_of?(String) && e.start_with?("stdio://")
               "stdio-tar://"
             elsif e == file
-              ::File.dirname(file)
+              dir = ::File.dirname(file)
+              dir[0] == '/' ? dir : "/#{dir}"
             else
               e
             end
@@ -565,4 +585,4 @@ module Etna
       end
     end
   end
-end
+end

data/lib/etna/instrumentation.rb

@@ -0,0 +1,56 @@
+module Etna
+  module Instrumentation
+    def self.included(cls)
+      cls.instance_eval do
+        def self.time_it(method_name, &metric_block)
+          orig_method_name = :"#{method_name}_without_time_it"
+          self.alias_method orig_method_name, method_name
+
+          self.define_method method_name do |*args|
+            time_it(orig_method_name, args, &metric_block)
+          end
+        end
+      end
+    end
+
+    def increment_it(&block)
+      if has_yabeda?
+        metric = yield
+        metric.increment({})
+      end
+    end
+
+    def time_it(method_name, args, &metric_block)
+      if has_yabeda?
+        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        begin
+          return send(method_name, *args)
+        ensure
+          dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+          if block_given?
+            metric = yield
+            tags = {}
+          else
+            tags = {class_name: self.class.name, method_name: method_name.to_s}
+            metric = Yabeda.etna.perf
+          end
+          metric.measure(tags, dur)
+        end
+      else
+        return send(method_name, *args, **kwds)
+      end
+    end
+
+    def with_yabeda_tags(tags, &block)
+      if has_yabeda?
+        Yabeda.with_tags(tags, &block)
+      else
+        yield
+      end
+    end
+
+    def has_yabeda?
+      defined?(Yabeda) && Yabeda.configured?
+    end
+  end
+end

data/lib/etna/route.rb

@@ -1,9 +1,13 @@
 require 'digest'
 require 'date'
+require "addressable/uri"
 require_relative "./censor"
+require_relative './instrumentation'
 
 module Etna
   class Route
+    include Etna::Instrumentation
+
     attr_reader :name
 
     def initialize(method, route, options, &block)
@@ -42,7 +46,7 @@ module Etna
       if params
         PARAM_TYPES.reduce(route) do |path,pat|
           path.gsub(pat) do
-            params[$1.to_sym].split('/').map { |c| block_given? ? yield(c) : URI.encode_www_form_component(c) }.join('/')
+            params[$1.to_sym].split('/').map { |c| block_given? ? yield(c) : Addressable::URI.normalized_encode(c) }.join('/')
           end
         end
       else
@@ -64,19 +68,27 @@ module Etna
     end
 
     def call(app, request)
-      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      update_params(request)
 
-      try_yabeda(request)  do |tags|
-        Yabeda.etna.visits.increment(tags)
+      tags = {}
+      if @action
+        tags[:controller], tags[:action] = @action.split('#')
+      elsif @name
+        tags[:action] = @name
+      else
+        tags[:action] = @route
       end
 
-      begin
+      params = request.env['rack.request.params']
+      user = request.env['etna.user']
+      tags[:user_hash] = user ? hash_user_email(user.email) : 'unknown'
+      if params && (params.include?(:project_name) || params.include?('project_name'))
+        tags[:project_name] = params[:project_name] || params['project_name']
+      end
+
+      with_yabeda_tags(tags) do
+        increment_it { Yabeda.etna.visits }
         process_call(app, request)
-      ensure
-        try_yabeda(request) do |tags|
-          dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
-          Yabeda.etna.response_time.measure(tags, dur)
-        end
       end
     end
 
@@ -93,36 +105,7 @@ module Etna
       Digest::MD5.hexdigest(digest)
     end
 
-    def try_yabeda(request, &block)
-      if @action
-        controller, action = @action.split('#')
-      elsif @name
-        controller = "none"
-        action = @name
-      else
-        controller = "none"
-        action = @route
-      end
-
-      params = request.env['rack.request.params']
-      user = request.env['etna.user']
-      user_hash = user ? hash_user_email(user.email) : 'unknown'
-      project_name = "unknown"
-
-      if params && (params.include?(:project_name) || params.include?('project_name'))
-        project_name = params[:project_name] || params['project_name']
-      end
-
-      begin
-        block.call({ controller: controller, action: action, user_hash: user_hash, project_name: project_name })
-      rescue => e
-        raise e unless Etna::Application.instance.environment == :production
-      end
-    end
-
     def process_call(app, request)
-      update_params(request)
-
       unless authorized?(request)
         if cc_available?(request)
           if request.content_type == 'application/json'
@@ -155,6 +138,8 @@ module Etna
       end
     end
 
+    time_it(:process_call) { Yabeda.etna.response_time }
+
     # the route does not require authorization
     def noauth?
       @auth && @auth[:noauth]
@@ -256,7 +241,7 @@ module Etna
         Hash[
           match.names.map(&:to_sym).zip(
             match.captures.map do |capture|
-              capture.split('/').map {|c| URI.decode_www_form_component(c) }.join('/')
+              capture.split('/').map {|c| Addressable::URI.unencode_component(c) }.join('/')
             end
           )
         ]

data/lib/etna/user.rb

@@ -71,5 +71,9 @@ module Etna
     def active? project=nil
       permissions.keys.length > 0
     end
+
+    def display_name
+      [ email, name ].join('|')
+    end
   end
 end

data/lib/etna.rb

@@ -28,6 +28,7 @@ require_relative './etna/metrics'
 require_relative './etna/remote'
 require_relative './etna/synchronize_db'
 require_relative './etna/janus_utils'
+require_relative './etna/instrumentation'
 
 class EtnaApp
   include Etna::Application

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.46
+  version: 0.1.47
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-08-02 00:00:00.000000000 Z
+date: 2023-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -200,6 +200,7 @@ files:
 - lib/etna/generate_autocompletion_script.rb
 - lib/etna/hmac.rb
 - lib/etna/injection.rb
+- lib/etna/instrumentation.rb
 - lib/etna/janus_utils.rb
 - lib/etna/json_serializable_struct.rb
 - lib/etna/logger.rb
@@ -240,8 +241,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.3
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Base classes for Mount Etna applications