etna 0.1.52 → 0.2.0

data/lib/etna/clients/magma/models.rb

@@ -13,8 +13,8 @@ module Etna
       class RetrievalRequest < Struct.new(:model_name, :attribute_names, :record_names, :project_name, :page, :page_size, :order, :filter, :hide_templates, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**params)
-          super({model_name: 'all', attribute_names: 'all', record_names: []}.update(params))
+        def initialize(params = {}, **kwds)
+          super(**{model_name: 'all', attribute_names: 'all', record_names: []}.update(params).update(kwds))
         end
       end
 
@@ -22,12 +22,12 @@ module Etna
         include JsonSerializableStruct
       end
 
-      class UpdateRequest < Struct.new(:revisions, :project_name, :dry_run, keyword_init: true)
+      class UpdateRequest < Struct.new(:revisions, :project_name, :dry_run, :autolink, keyword_init: true)
         include JsonSerializableStruct
         include MultipartSerializableNestedHash
 
-        def initialize(**params)
-          super({revisions: {}, dry_run: false}.update(params))
+        def initialize(params = {}, **kwds)
+          super(**{revisions: {}, dry_run: false}.update(params).update(kwds))
         end
 
         def update_revision(model_name, record_name, attrs)
@@ -49,8 +49,8 @@ module Etna
       class UpdateModelRequest < Struct.new(:project_name, :actions, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**params)
-          super({actions: []}.update(params))
+        def initialize(params = {}, **kwds)
+          super(**{actions: []}.update(params).update(kwds))
         end
 
         def add_action(action)
@@ -58,27 +58,35 @@ module Etna
         end
       end
 
-      class AddModelAction < Struct.new(:action_name, :model_name, :parent_model_name, :parent_link_type, :identifier, :date_shift_root, keyword_init: true)
+      class AddModelAction < Struct.new(:action_name, :model_name, :parent_model_name, :parent_link_type, :identifier, :date_shift_root, :template_project_name, :template_model_name, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'add_model', date_shift_root: false}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'add_model', date_shift_root: false}.update(args).update(kwds))
         end
       end
 
       class SetDateShiftRootAction < Struct.new(:action_name, :model_name, :date_shift_root, keyword_init: true)
         include JsonSerializableStruct
 
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'set_date_shift_root'}.update(args).update(kwds))
+        end
+      end
+
+      class SetModelTemplateAction < Struct.new(:action_name, :model_name, :template_project_name, :template_model_name, :clear_template, keyword_init: true)
+        include JsonSerializableStruct
+
         def initialize(**args)
-          super({action_name: 'set_date_shift_root'}.update(args))
+          super({action_name: 'set_model_template', clear_template: false}.update(args))
         end
       end
 
       class AddAttributeAction < Struct.new(:action_name, :model_name, :attribute_name, :type, :description, :display_name, :format_hint, :hidden, :index, :link_model_name, :read_only, :attribute_group, :restricted, :unique, :validation, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'add_attribute'}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'add_attribute'}.update(args).update(kwds))
         end
 
         def attribute_type=(val)
@@ -101,8 +109,8 @@ module Etna
       class AddLinkAction < Struct.new(:action_name, :links, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'add_link', links: []}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'add_link', links: []}.update(args).update(kwds))
         end
       end
 
@@ -113,16 +121,16 @@ module Etna
       class AddProjectAction < Struct.new(:action_name, :no_metis_bucket, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'add_project'}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'add_project'}.update(args).update(kwds))
         end
       end
 
       class UpdateAttributeAction < Struct.new(:action_name, :model_name, :attribute_name, :description, :display_name, :format_hint, :hidden, :index, :link_model_name, :read_only, :attribute_group, :restricted, :unique, :validation, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'update_attribute'}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'update_attribute'}.update(args).update(kwds))
         end
 
         def as_json
@@ -133,8 +141,8 @@ module Etna
       class RenameAttributeAction < Struct.new(:action_name, :model_name, :attribute_name, :new_attribute_name, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'rename_attribute'}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'rename_attribute'}.update(args).update(kwds))
         end
       end
 
@@ -159,6 +167,12 @@ module Etna
         def models
           Models.new(raw['models'])
         end
+
+        def update(response)
+          response.models.each do |model_name, model|
+            models.update( model_name, model )
+          end
+        end
       end
 
       class UpdateModelResponse < RetrievalResponse
@@ -219,6 +233,12 @@ module Etna
           Model.new(raw[model_key.to_s])
         end
 
+        def each
+          model_keys.each do |model_key|
+            yield model_key, model(model_key)
+          end
+        end
+
         def all
           raw.values.map { |r| Model.new(r) }
         end
@@ -229,6 +249,20 @@ module Etna
           Models.new({}.update(raw).update(raw_update))
         end
 
+        def update(model_name, model)
+          raw[ model_name.to_s ] ||= {}
+          raw[ model_name.to_s ]["template"] = model.template.raw
+          raw[ model_name.to_s ]["documents"] ||= {}
+          raw[ model_name.to_s ]["documents"].update(model.documents.raw)
+        end
+
+        def is_table?(model_name)
+          parent_model_name = self.model(model_name)&.template&.parent
+          parent_model = self.model(parent_model_name)
+          parent_attribute = parent_model&.template&.attributes&.attribute(model_name)
+          parent_attribute&.attribute_type == 'table'
+        end
+
         # Can look up reciprocal links by many means.  At minimum, a source model or model name must be provided,
         # and either a link_attribute_name, attribute, or link_model.
         def find_reciprocal(
@@ -238,7 +272,7 @@ module Etna
             attribute: model&.template&.attributes&.attribute(link_attribute_name),
             link_model: self.model(attribute&.link_model_name)
         )
-          return nil if model.nil? || model.name.nil?
+          return nil if model.nil?
 
           return link_model&.template&.attributes&.all&.find { |a| a.name == link_attribute_name } if link_attribute_name
 
@@ -371,6 +405,14 @@ module Etna
           Attributes.new(raw['attributes'] ||= {})
         end
 
+        def template_project_name
+          raw['template_project_name']
+        end
+
+        def template_model_name
+          raw['template_model_name']
+        end
+
         def build_attributes
           Attributes.new(raw['attributes'] ||= {})
         end

data/lib/etna/clients/magma/workflows/model_synchronization_workflow.rb

@@ -249,10 +249,8 @@ module Etna
           template = source_model.template
 
           add_model_action = AddModelAction.new(
-              {
-                  model_name: target_model_name,
-                  identifier: template.identifier,
-              }
+              model_name: target_model_name,
+              identifier: template.identifier,
           )
 
           parents = template.attributes.all.select { |a| a.attribute_type == AttributeType::PARENT }

data/lib/etna/clients/metis/client.rb

@@ -23,6 +23,16 @@ module Etna
           @etna_client.folder_list_all_folders(list_all_folders_request.to_h))
       end
 
+      def tail_bucket(tail_bucket_request = TailBucketRequest.new)
+        TailResponse.new(
+          tail_bucket_request.project_name,
+          tail_bucket_request.bucket_name,
+          @etna_client.bucket_tail(tail_bucket_request.to_h).split("\n").map do |line|
+            JSON.parse(line, symbolize_names: true)
+          end
+        )
+      end
+
       def list_folder(list_folder_request = ListFolderRequest.new)
         if list_folder_request.folder_path != ''
           FoldersAndFilesResponse.new(
@@ -101,16 +111,36 @@ module Etna
 
       def download_file(file_or_url = File.new, &block)
         if file_or_url.instance_of?(File)
-          download_path =  file_or_url.download_path
+          download_url = file_or_url.download_url
+
+          if !download_url.present?
+            download = authorize_download(AuthorizeDownloadRequest.new(
+              project_name: file_or_url.project_name,
+              bucket_name: file_or_url.bucket_name,
+              file_path: file_or_url.file_path
+            ))
+            download_url = download.download_url
+          end
         else
-          download_path = file_or_url.sub(%r!^https://[^/]*?/!, '/')
+          download_url = file_or_url
         end
 
+        download_path = download_url.sub(%r!^https://[^/]*?/!, '/')
+
         @etna_client.get(download_path) do |response|
           response.read_body(&block)
         end
       end
 
+      def authorize_download(authorize_download_request = AuthorizeDownloadRequest.new)
+        json = nil
+        @etna_client.post("/authorize/download", authorize_download_request) do |res|
+          json = JSON.parse(res.body)
+        end
+
+        DownloadResponse.new(json)
+      end
+
       def file_metadata(file_or_url = File.new)
         if file_or_url.instance_of?(File)
           download_path =  file_or_url.download_path
@@ -329,11 +359,45 @@ module Etna
         ))
       end
 
+      def get_file_count_by_project(get_file_count_by_project_request = GetFileCountByProjectRequest.new)
+        query = ""
+        unless get_file_count_by_project_request.project_names.nil?
+          query = "?#{create_query_list_str('projects', get_file_count_by_project_request.project_names)}"
+        end
+
+        json = nil
+        @etna_client.get("/api/stats/files#{query}") do |res|
+          json = JSON.parse(res.body, symbolize_names: true)
+        end
+
+        json
+      end
+
+      def get_byte_count_by_project(get_byte_count_by_project_request = GetByteCountByProjectRequest.new)
+        query = ""
+        unless get_byte_count_by_project_request.project_names.nil?
+          query = "?#{create_query_list_str('projects', get_byte_count_by_project_request.project_names)}"
+        end
+
+        json = nil
+        @etna_client.get("/api/stats/bytes#{query}") do |res|
+          json = JSON.parse(res.body, symbolize_names: true)
+        end
+
+        json
+      end
+
       private
 
       def parent_folder_path(folder_path)
         folder_path.split('/')[0..-2].join('/')
       end
+
+      def create_query_list_str(name, values)
+        values.map do |val|
+          "#{name}[]=#{val}"
+        end.join('&')
+      end
     end
   end
 end

data/lib/etna/clients/metis/models.rb

@@ -158,6 +158,21 @@ module Etna
         end
       end
 
+      class TailBucketRequest < Struct.new(:project_name, :bucket_name, :type, :folder_id, :batch_start, :batch_end, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
       class FindRequest < Struct.new(:project_name, :bucket_name, :limit, :offset, :params, :hide_paths, keyword_init: true)
         include JsonSerializableStruct
 
@@ -219,25 +234,99 @@ module Etna
         end
       end
 
-      class FoldersResponse
-        attr_reader :raw
+      class TailNode < Etna::Clients::Response
+        def initialize(tail, raw = {})
+          super(raw)
+          @tail = tail
+        end
 
-        def initialize(raw = {})
-          @raw = raw
+        def parent?
+          @raw[:type] == 'parent'
         end
 
-        def folders
-          Folders.new(raw[:folders])
+        def folder?
+          @raw[:type] == 'folder'
+        end
+
+        def file?
+          @raw[:type] == 'file'
+        end
+
+        params :id, :node_name, :parent_id, :file_hash, :updated_at
+
+        def as_file
+          {
+            file_name: node_name,
+            file_hash: file_hash,
+            updated_at: updated_at,
+            file_path: file_path,
+            folder_id: parent_id,
+            project_name: @tail.project_name,
+            bucket_name: @tail.bucket_name,
+          }
+        end
+
+        def parent_path
+          return nil if !parent_id
+
+          if !@tail.paths[parent_id]
+            parent = @tail.parents[parent_id]
+            @tail.paths[parent_id] = parent.file_path
+          end
+
+          return @tail.paths[parent_id]
+        end
+
+        def file_path
+          [ parent_path, node_name ].compact.join('/')
         end
       end
 
-      class FilesResponse
-        attr_reader :raw
+      class TailResponse < Etna::Clients::Response
+        attr_reader :project_name, :bucket_name, :paths, :parents
+
+
+        def initialize(project_name, bucket_name, raw = {})
+          super(raw)
+
+          @paths = {}
+
+          @parents = {}
+
+          @project_name = project_name
+          @bucket_name = bucket_name
+
+          @nodes = []
+          raw.each do |raw_node|
+            node = TailNode.new(self, raw_node)
+            if node.parent?
+              @parents[node.id] = node
+            else
+              @nodes.push(node)
+            end
+          end
+        end
+
+        def folders
+          @nodes.filter_map do |node|
+            Folder.new(node.as_folder) if node.folder?
+          end
+        end
+
+        def files
+          @nodes.filter_map do |node|
+            File.new(node.as_file) if node.file?
+          end
+        end
+      end
 
-        def initialize(raw = {})
-          @raw = raw
+      class FoldersResponse < Etna::Clients::Response
+        def folders
+          Folders.new(raw[:folders])
         end
+      end
 
+      class FilesResponse < Etna::Clients::Response
         def files
           Files.new(raw[:files])
         end
@@ -253,37 +342,19 @@ module Etna
         end
       end
 
-      class Files
-        attr_reader :raw
-
-        def initialize(raw = {})
-          @raw = raw
-        end
-
+      class Files < Etna::Clients::Response
         def all
           raw.map { |file| File.new(file) }
         end
       end
 
-      class Folders
-        attr_reader :raw
-
-        def initialize(raw = {})
-          @raw = raw
-        end
-
+      class Folders < Etna::Clients::Response
         def all
           raw.map { |folder| Folder.new(folder) }
         end
       end
 
-      class File
-        attr_reader :raw
-
-        def initialize(raw = {})
-          @raw = raw
-        end
-
+      class File < Etna::Clients::Response
         def with_containing_folder(folder)
           folder_path = folder.is_a?(Folder) ? folder.folder_path : folder
           File.new({}.update(self.raw).update({
@@ -291,17 +362,7 @@ module Etna
           }))
         end
 
-        def file_path
-          raw[:file_path]
-        end
-
-        def project_name
-          raw[:project_name]
-        end
-
-        def bucket_name
-          raw[:bucket_name]
-        end
+        params :file_path, :project_name, :bucket_name, :file_name, :size, :file_hash, :folder_id
 
         def download_path
           raw[:download_url].nil? ?
@@ -313,65 +374,40 @@ module Etna
           raw[:download_url] || ''
         end
 
-        def file_name
-          raw[:file_name]
-        end
-
         def updated_at
           time = raw[:updated_at]
           time.nil? ? nil : Time.parse(time)
         end
 
-        def size
-          raw[:size]
+        def as_magma_file_attribute
+          {
+            path: as_metis_url,
+            original_filename: file_name || ::File.basename(file_path)
+          }
         end
 
-        def file_hash
-          raw[:file_hash]
-        end
-
-        def folder_id
-          raw[:folder_id]
+        def as_metis_url
+          "metis://#{project_name}/#{bucket_name}/#{file_path}"
         end
       end
 
-      class Folder
-        attr_reader :raw
-
-        def initialize(raw = {})
-          @raw = raw
-        end
-
-        def folder_path
-          raw[:folder_path]
-        end
-
-        def folder_name
-          raw[:folder_name]
-        end
-
-        def bucket_name
-          raw[:bucket_name]
-        end
-
-        def project_name
-          raw[:project_name]
-        end
+      class Folder < Etna::Clients::Response
+        params :folder_path, :folder_name, :bucket_name, :project_name, :id
 
         def updated_at
           time = raw[:updated_at]
           time.nil? ? nil : Time.parse(time)
         end
-
-        def id
-          raw[:id]
-        end
       end
 
       class AuthorizeUploadRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)
         include JsonSerializableStruct
       end
 
+      class AuthorizeDownloadRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)
+        include JsonSerializableStruct
+      end
+
       class UploadStartRequest < Struct.new(:file_size, :action, :metis_uid, :next_blob_size, :upload_path, :next_blob_hash, :reset, keyword_init: true)
         include JsonSerializableStruct
 
@@ -392,12 +428,13 @@ module Etna
         end
       end
 
-      class UploadResponse
-        attr_reader :raw
-        def initialize(raw = {})
-          @raw = raw
+      class DownloadResponse < Etna::Clients::Response
+        def download_url
+          raw['download_url']
         end
+      end
 
+      class UploadResponse < Etna::Clients::Response
         def current_byte_position
           raw['current_byte_position'].to_i
         end
@@ -419,6 +456,22 @@ module Etna
         START = UploadAction.new("start")
         BLOB = UploadAction.new("blob")
       end
+
+      class GetFileCountByProjectRequest < Struct.new(:project_names, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+      end
+
+      class GetByteCountByProjectRequest < Struct.new(:project_names, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+      end
     end
   end
-end
+end

data/lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb

@@ -9,24 +9,34 @@ module Etna
         # Since we are doing manual triage of files,
         #   do not automatically copy directory trees.
         # srcs must be a list of full paths to files.
-        def copy_files(srcs, &block)
-          srcs.each do |src|
+        def copy_files(files, &block)
+          files.each do |file|
+            if file.is_a?(Array)
+              src, dest = file
+            else
+              src = dest = file
+            end
+
             if !ingest_filesystem.exist?(src)
               logger&.warn("#{src} does not exist on source filesystem. Skipping.")
+              yield src, false if block_given?
               next
             end
 
             logger&.info("Copying file #{src} (#{Etna::Formatting.as_size(ingest_filesystem.stat(src).size)})")
 
-            # For ingestion triage, just copy over the exact path + filename.
-            copy_file(dest: src, src: src, &block)
+            begin
+              copy_file(dest: dest, src: src, &block)
+            rescue Exception => e
+              yield src, false if block_given?
+            end
           end
         end
 
         def copy_file(dest:, src:, &block)
           ingest_filesystem.with_readable(src, "r") do |io|
             metis_filesystem.do_streaming_upload(io, dest, ingest_filesystem.stat(src).size)
-            yield src if block_given?
+            yield src, true if block_given?
           end
         end
       end

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -197,6 +197,7 @@ module Etna
                 end
               end
             end
+            @last_bytes
           end
         end
       end

data/lib/etna/clients/models.rb

@@ -0,0 +1,19 @@
+module Etna
+  module Clients
+    class Response
+      attr_reader :raw
+
+      def self.params(*params)
+        params.each do |param|
+          self.define_method param do
+            @raw[param]
+          end
+        end
+      end
+
+      def initialize(raw = {})
+        @raw = raw || {}
+      end
+    end
+  end
+end