etna 0.1.51 → 0.2.0

data/lib/etna/clients/gnomon/models.rb

@@ -0,0 +1,33 @@
+require_relative '../base_client'
+
+# TODO:  In the near future, I'd like to transition to specifying apis via SWAGGER and generating model stubs from the
+# common definitions.  For nowe I've written them out by hand here.
+module Etna
+  module Clients
+    class Gnomon < Etna::Clients::BaseClient
+      class ProjectRulesResponse
+        attr_reader :raw
+
+        def initialize(raw = {})
+          @raw = raw
+        end
+
+        def rules
+          Rules.new(raw['rules'])
+        end
+      end
+
+      class Rules
+        attr_reader :raw
+
+        def initialize(raw = {})
+          @raw = raw
+        end
+
+        def [](model_name)
+          raw[model_name.to_s]
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/gnomon.rb

@@ -0,0 +1,2 @@
+require_relative './gnomon/client'
+require_relative './gnomon/models'

data/lib/etna/clients/janus/client.rb

@@ -40,6 +40,14 @@ module Etna
         end
       end
 
+      def update_project(update_project_request = UpdateProjectRequest.new)
+        @etna_client.post(
+          "/api/admin/#{update_project_request.project_name}/update",
+          update_project_request) do |res|
+          # Redirect, no response data
+        end
+      end
+
       def update_permission(update_permission_request = UpdatePermissionRequest.new)
         @etna_client.post(
           "/api/admin/#{update_permission_request.project_name}/update_permission",
@@ -74,6 +82,24 @@ module Etna
 
         response.body
       end
+
+      def get_project_stats(get_stats_request = GetStatsRequest.new)
+        query = ""
+        unless get_stats_request.project_names.nil?
+          query = "?"
+          query += get_stats_request.project_names.map do |name|
+            "projects[]=#{name}"
+          end.join('&')
+        end
+
+        json = nil
+
+        @etna_client.get("/api/stats/projects#{query}") do |res|
+          json = JSON.parse(res.body, symbolize_names: true)
+        end
+
+        json
+      end
     end
   end
 end

data/lib/etna/clients/janus/models.rb

@@ -31,6 +31,14 @@ module Etna
         end
       end
 
+      class UpdateProjectRequest < Struct.new(:project_name, :resource, :requires_agreement, :cc_text, :contact_email, :project_name_full, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+      end
+
       class UpdatePermissionRequest < Struct.new(:project_name, :email, :role, :privileged, :affiliation, keyword_init: true)
         include JsonSerializableStruct
 
@@ -132,6 +140,14 @@ module Etna
           @raw
         end
       end
+
+      class GetStatsRequest < Struct.new(:project_names, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+      end
     end
   end
 end

data/lib/etna/clients/janus/workflows/generate_token_workflow.rb

@@ -25,7 +25,7 @@ module Etna
             until private_key_file
               puts "Location of private key file?"
               private_key_file = ::File.expand_path(STDIN.gets.chomp)
-              unless File.exists?(private_key_file)
+              unless ::File.exist?(private_key_file)
                 puts "No such file."
                 private_key_file = nil
               end

data/lib/etna/clients/magma/client.rb

@@ -56,9 +56,13 @@ module Etna
       #   attribute_names:  "all"
       # }
       def retrieve(retrieval_request = RetrievalRequest.new)
+        if retrieval_request.is_a?(Hash)
+          retrieval_request = RetrievalRequest.new(retrieval_request)
+        end
+
         json = nil
         @etna_client.post('/retrieve', retrieval_request) do |res|
-          json = JSON.parse(res.body)
+          json = safe_parse(res)
         end
 
         RetrievalResponse.new(json)
@@ -69,7 +73,7 @@ module Etna
       def query(query_request = QueryRequest.new)
         json = nil
         @etna_client.post('/query', query_request) do |res|
-          json = JSON.parse(res.body)
+          json = safe_parse(res)
         end
 
         QueryResponse.new(json)
@@ -77,17 +81,124 @@ module Etna
 
       def update(update_request = UpdateRequest.new)
         json = nil
+
         @etna_client.multipart_post('/update', update_request.encode_multipart_content) do |res|
-          json = JSON.parse(res.body)
+          json = safe_parse(res)
         end
 
         UpdateResponse.new(json)
       end
 
-      def update_json(update_request = UpdateRequest.new)
+      class Pager
+        def initialize(client, update_request, page_size)
+          @client = client
+
+          @update_request = update_request
+
+          @page_size = [ page_size || 2, 2 ].max
+        end
+
+        def response
+          page_count = 0
+          page = {}
+
+          results = UpdateResponse.new({"models" => {}})
+
+          flat_records.each do |flat_record|
+            page_count += 1 + flat_record[:tables].length
+
+            add_to_page( page, flat_record )
+
+            flat_record[:tables].each do |table_record|
+              add_to_page(page, table_record)
+            end
+
+            if page_count >= @page_size
+              results.update(update_part(page))
+              page = {}
+              page_count = 0
+            end
+          end
+
+          results.update(update_part(page)) if page_count > 0
+
+          return results
+        end
+
+        def revisions
+          @update_request.revisions
+        end
+
+        def flat_records
+          project = @client.retrieve(
+            project_name: @update_request.project_name,
+            hide_templates: false
+          )
+
+          flat_records = {}
+          revisions.each do |model_name, records|
+            records.each do |record_name, record|
+              flat = {
+                  model_name: model_name,
+                  record_name: record_name,
+                  record: record
+              }
+              flat_name = "#{model_name}.#{record_name}"
+
+              # if the update sets a table attribute, attach table records to their parent so they are submitted together
+              flat_tables = []
+              record.each do |attribute_name, value|
+                att = project.models.model(model_name).template.attributes.attribute(attribute_name)
+                if att.attribute_type == 'table'
+                  value.each do |table_entry_name|
+                    table_record = revisions[ att.link_model_name ][table_entry_name]
+                    flat_tables.push(
+                      {
+                        model_name: att.link_model_name,
+                        record_name: table_entry_name,
+                        record: table_record
+                      }
+                    )
+
+                    # exclude the attached table record from the flattened list of records
+                    flat_records[ "#{ att.link_model_name }.#{table_entry_name}" ] = nil
+                  end
+                end
+              end
+              flat[:tables] = flat_tables
+              flat_records[ flat_name ] = flat unless flat_records.has_key?(flat_name)
+            end
+          end
+
+          return flat_records.values.compact.shuffle
+        end
+
+        def update_part(page)
+          @client.update_json(
+            UpdateRequest.new(
+              project_name: @update_request.project_name,
+              revisions: page,
+              dry_run: @update_request.dry_run,
+              autolink: @update_request.autolink
+            )
+          )
+        end
+
+        def add_to_page(page, entry)
+          page[ entry[:model_name] ] ||= {}
+          page[ entry[:model_name] ][ entry[:record_name] ] = entry[:record]
+        end
+      end
+
+      def update_json(update_request = UpdateRequest.new, page_size=nil)
         json = nil
+
+        if page_size
+          return Etna::Clients::Magma::Pager.new(self, update_request, page_size).response
+        end
+
         @etna_client.post('/update', update_request) do |res|
-          json = JSON.parse(res.body)
+          json = safe_parse(res)
         end
 
         UpdateResponse.new(json)
@@ -96,11 +207,20 @@ module Etna
       def update_model(update_model_request = UpdateModelRequest.new)
         json = nil
         @etna_client.post('/update_model', update_model_request) do |res|
-          json = JSON.parse(res.body)
+          json = safe_parse(res)
         end
 
         UpdateModelResponse.new(json)
       end
+
+      def set_flags(project_name:, flags:)
+        json = nil
+        @etna_client.post("/flags/#{project_name}", { flags: flags }) do |res|
+          json = safe_parse(res)
+        end
+
+        json
+      end
     end
   end
 end

data/lib/etna/clients/magma/models.rb

@@ -13,8 +13,8 @@ module Etna
       class RetrievalRequest < Struct.new(:model_name, :attribute_names, :record_names, :project_name, :page, :page_size, :order, :filter, :hide_templates, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**params)
-          super({model_name: 'all', attribute_names: 'all', record_names: []}.update(params))
+        def initialize(params = {}, **kwds)
+          super(**{model_name: 'all', attribute_names: 'all', record_names: []}.update(params).update(kwds))
         end
       end
 
@@ -22,12 +22,12 @@ module Etna
         include JsonSerializableStruct
       end
 
-      class UpdateRequest < Struct.new(:revisions, :project_name, :dry_run, keyword_init: true)
+      class UpdateRequest < Struct.new(:revisions, :project_name, :dry_run, :autolink, keyword_init: true)
         include JsonSerializableStruct
         include MultipartSerializableNestedHash
 
-        def initialize(**params)
-          super({revisions: {}, dry_run: false}.update(params))
+        def initialize(params = {}, **kwds)
+          super(**{revisions: {}, dry_run: false}.update(params).update(kwds))
         end
 
         def update_revision(model_name, record_name, attrs)
@@ -49,8 +49,8 @@ module Etna
       class UpdateModelRequest < Struct.new(:project_name, :actions, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**params)
-          super({actions: []}.update(params))
+        def initialize(params = {}, **kwds)
+          super(**{actions: []}.update(params).update(kwds))
         end
 
         def add_action(action)
@@ -58,27 +58,35 @@ module Etna
         end
       end
 
-      class AddModelAction < Struct.new(:action_name, :model_name, :parent_model_name, :parent_link_type, :identifier, :date_shift_root, keyword_init: true)
+      class AddModelAction < Struct.new(:action_name, :model_name, :parent_model_name, :parent_link_type, :identifier, :date_shift_root, :template_project_name, :template_model_name, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'add_model', date_shift_root: false}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'add_model', date_shift_root: false}.update(args).update(kwds))
         end
       end
 
       class SetDateShiftRootAction < Struct.new(:action_name, :model_name, :date_shift_root, keyword_init: true)
         include JsonSerializableStruct
 
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'set_date_shift_root'}.update(args).update(kwds))
+        end
+      end
+
+      class SetModelTemplateAction < Struct.new(:action_name, :model_name, :template_project_name, :template_model_name, :clear_template, keyword_init: true)
+        include JsonSerializableStruct
+
         def initialize(**args)
-          super({action_name: 'set_date_shift_root'}.update(args))
+          super({action_name: 'set_model_template', clear_template: false}.update(args))
         end
       end
 
       class AddAttributeAction < Struct.new(:action_name, :model_name, :attribute_name, :type, :description, :display_name, :format_hint, :hidden, :index, :link_model_name, :read_only, :attribute_group, :restricted, :unique, :validation, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'add_attribute'}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'add_attribute'}.update(args).update(kwds))
         end
 
         def attribute_type=(val)
@@ -101,8 +109,8 @@ module Etna
       class AddLinkAction < Struct.new(:action_name, :links, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'add_link', links: []}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'add_link', links: []}.update(args).update(kwds))
         end
       end
 
@@ -113,16 +121,16 @@ module Etna
       class AddProjectAction < Struct.new(:action_name, :no_metis_bucket, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'add_project'}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'add_project'}.update(args).update(kwds))
         end
       end
 
       class UpdateAttributeAction < Struct.new(:action_name, :model_name, :attribute_name, :description, :display_name, :format_hint, :hidden, :index, :link_model_name, :read_only, :attribute_group, :restricted, :unique, :validation, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'update_attribute'}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'update_attribute'}.update(args).update(kwds))
         end
 
         def as_json
@@ -133,8 +141,8 @@ module Etna
       class RenameAttributeAction < Struct.new(:action_name, :model_name, :attribute_name, :new_attribute_name, keyword_init: true)
         include JsonSerializableStruct
 
-        def initialize(**args)
-          super({action_name: 'rename_attribute'}.update(args))
+        def initialize(args = {}, **kwds)
+          super(**{action_name: 'rename_attribute'}.update(args).update(kwds))
         end
       end
 
@@ -159,6 +167,12 @@ module Etna
         def models
           Models.new(raw['models'])
         end
+
+        def update(response)
+          response.models.each do |model_name, model|
+            models.update( model_name, model )
+          end
+        end
       end
 
       class UpdateModelResponse < RetrievalResponse
@@ -219,6 +233,12 @@ module Etna
           Model.new(raw[model_key.to_s])
         end
 
+        def each
+          model_keys.each do |model_key|
+            yield model_key, model(model_key)
+          end
+        end
+
         def all
           raw.values.map { |r| Model.new(r) }
         end
@@ -229,6 +249,20 @@ module Etna
           Models.new({}.update(raw).update(raw_update))
         end
 
+        def update(model_name, model)
+          raw[ model_name.to_s ] ||= {}
+          raw[ model_name.to_s ]["template"] = model.template.raw
+          raw[ model_name.to_s ]["documents"] ||= {}
+          raw[ model_name.to_s ]["documents"].update(model.documents.raw)
+        end
+
+        def is_table?(model_name)
+          parent_model_name = self.model(model_name)&.template&.parent
+          parent_model = self.model(parent_model_name)
+          parent_attribute = parent_model&.template&.attributes&.attribute(model_name)
+          parent_attribute&.attribute_type == 'table'
+        end
+
         # Can look up reciprocal links by many means.  At minimum, a source model or model name must be provided,
         # and either a link_attribute_name, attribute, or link_model.
         def find_reciprocal(
@@ -238,7 +272,7 @@ module Etna
             attribute: model&.template&.attributes&.attribute(link_attribute_name),
             link_model: self.model(attribute&.link_model_name)
         )
-          return nil if model.nil? || model.name.nil?
+          return nil if model.nil?
 
           return link_model&.template&.attributes&.all&.find { |a| a.name == link_attribute_name } if link_attribute_name
 
@@ -371,6 +405,14 @@ module Etna
           Attributes.new(raw['attributes'] ||= {})
         end
 
+        def template_project_name
+          raw['template_project_name']
+        end
+
+        def template_model_name
+          raw['template_model_name']
+        end
+
         def build_attributes
           Attributes.new(raw['attributes'] ||= {})
         end

data/lib/etna/clients/magma/workflows/model_synchronization_workflow.rb

@@ -249,10 +249,8 @@ module Etna
           template = source_model.template
 
           add_model_action = AddModelAction.new(
-              {
-                  model_name: target_model_name,
-                  identifier: template.identifier,
-              }
+              model_name: target_model_name,
+              identifier: template.identifier,
           )
 
           parents = template.attributes.all.select { |a| a.attribute_type == AttributeType::PARENT }

data/lib/etna/clients/metis/client.rb

@@ -23,6 +23,16 @@ module Etna
           @etna_client.folder_list_all_folders(list_all_folders_request.to_h))
       end
 
+      def tail_bucket(tail_bucket_request = TailBucketRequest.new)
+        TailResponse.new(
+          tail_bucket_request.project_name,
+          tail_bucket_request.bucket_name,
+          @etna_client.bucket_tail(tail_bucket_request.to_h).split("\n").map do |line|
+            JSON.parse(line, symbolize_names: true)
+          end
+        )
+      end
+
       def list_folder(list_folder_request = ListFolderRequest.new)
         if list_folder_request.folder_path != ''
           FoldersAndFilesResponse.new(
@@ -101,16 +111,36 @@ module Etna
 
       def download_file(file_or_url = File.new, &block)
         if file_or_url.instance_of?(File)
-          download_path =  file_or_url.download_path
+          download_url = file_or_url.download_url
+
+          if !download_url.present?
+            download = authorize_download(AuthorizeDownloadRequest.new(
+              project_name: file_or_url.project_name,
+              bucket_name: file_or_url.bucket_name,
+              file_path: file_or_url.file_path
+            ))
+            download_url = download.download_url
+          end
         else
-          download_path = file_or_url.sub(%r!^https://[^/]*?/!, '/')
+          download_url = file_or_url
         end
 
+        download_path = download_url.sub(%r!^https://[^/]*?/!, '/')
+
         @etna_client.get(download_path) do |response|
           response.read_body(&block)
         end
       end
 
+      def authorize_download(authorize_download_request = AuthorizeDownloadRequest.new)
+        json = nil
+        @etna_client.post("/authorize/download", authorize_download_request) do |res|
+          json = JSON.parse(res.body)
+        end
+
+        DownloadResponse.new(json)
+      end
+
       def file_metadata(file_or_url = File.new)
         if file_or_url.instance_of?(File)
           download_path =  file_or_url.download_path
@@ -329,11 +359,45 @@ module Etna
         ))
       end
 
+      def get_file_count_by_project(get_file_count_by_project_request = GetFileCountByProjectRequest.new)
+        query = ""
+        unless get_file_count_by_project_request.project_names.nil?
+          query = "?#{create_query_list_str('projects', get_file_count_by_project_request.project_names)}"
+        end
+
+        json = nil
+        @etna_client.get("/api/stats/files#{query}") do |res|
+          json = JSON.parse(res.body, symbolize_names: true)
+        end
+
+        json
+      end
+
+      def get_byte_count_by_project(get_byte_count_by_project_request = GetByteCountByProjectRequest.new)
+        query = ""
+        unless get_byte_count_by_project_request.project_names.nil?
+          query = "?#{create_query_list_str('projects', get_byte_count_by_project_request.project_names)}"
+        end
+
+        json = nil
+        @etna_client.get("/api/stats/bytes#{query}") do |res|
+          json = JSON.parse(res.body, symbolize_names: true)
+        end
+
+        json
+      end
+
       private
 
       def parent_folder_path(folder_path)
         folder_path.split('/')[0..-2].join('/')
       end
+
+      def create_query_list_str(name, values)
+        values.map do |val|
+          "#{name}[]=#{val}"
+        end.join('&')
+      end
     end
   end
 end