superset 0.2.6 → 0.3.0

data/lib/superset/dataset/list.rb

@@ -3,12 +3,12 @@ module Superset
     class List < Superset::Request
       attr_reader :title_contains, :title_equals, :schema_equals, :database_id_eq
 
-      def initialize(page_num: 0, title_contains: '', title_equals: '', schema_equals: '', database_id_eq: '')
+      def initialize(title_contains: '', title_equals: '', schema_equals: '', database_id_eq: '', **kwargs)
         @title_contains = title_contains
         @title_equals = title_equals
         @schema_equals = schema_equals
         @database_id_eq = database_id_eq
-        super(page_num: page_num)
+        super(**kwargs)
       end
 
       def self.call

data/lib/superset/dataset/put.rb

@@ -2,7 +2,7 @@
 #
 # Usage:
 # params = { owners: [ 58, 3 ] }
-# Superset::Dataset::Put.new(object_id: 101, params: params ).perform
+# Superset::Dataset::Put.new(target_id: 101, params: params ).perform
 
 module Superset
   module Dataset
@@ -11,7 +11,7 @@ module Superset
       private
 
       def route
-        "dataset/#{object_id}"
+        "dataset/#{target_id}"
       end
     end
   end

data/lib/superset/dataset/update_schema.rb

@@ -2,12 +2,13 @@ module Superset
   module Dataset
     class UpdateSchema < Superset::Request
 
-      attr_reader :source_dataset_id, :target_database_id, :target_schema, :remove_copy_suffix
+      attr_reader :source_dataset_id, :target_database_id, :target_schema, :target_catalog, :remove_copy_suffix
 
-      def initialize(source_dataset_id: , target_database_id: , target_schema: , remove_copy_suffix: false)
+      def initialize(source_dataset_id: , target_database_id: , target_schema: , target_catalog: nil,  remove_copy_suffix: false)
         @source_dataset_id = source_dataset_id
         @target_database_id = target_database_id
         @target_schema = target_schema
+        @target_catalog = target_catalog
         @remove_copy_suffix = remove_copy_suffix
       end
 
@@ -37,6 +38,7 @@ module Superset
           
           # primary database and schema changes
           new_params.merge!("database_id": target_database_id)  # add the target database id
+          new_params['catalog'] = target_catalog
           new_params['schema'] = target_schema
           new_params['owners'] = new_params['owners'].map {|o| o['id'] } # expects an array of user ids
           new_params['table_name'] = new_params['table_name'].gsub(/ \(COPY\)/, '') if remove_copy_suffix

data/lib/superset/display.rb

@@ -7,28 +7,37 @@ module Superset
     def table
       Terminal::Table.new(
         title: title,
-        headings: headings,
+        headings: list_attributes.map(&:to_s).map(&:humanize),
         rows: rows
       )
     end
 
     def rows
-      result.map do |d|
-        list_attributes.map { |la| d[la].to_s }
+      if result.is_a?(Hash)
+        list_attributes.map { |la| result[la].to_s }
+      else
+        result.map do |d|
+          list_attributes.map { |la| d[la].to_s }
+        end
       end
     end
 
-    def title
-      self.class.to_s
+    def to_h
+      if result.is_a?(Hash)
+        list_attributes.to_h { |la| [la, result[la]] }
+      else
+        result.map do |d|
+          list_attributes.to_h { |la| [la, d[la]] }
+        end
+      end
     end
 
-    def headings
-      headings = display_headers ? display_headers : list_attributes
-      headings.map(&:to_s).map(&:humanize)
+    def ids
+      result.map { |d| d[:id] }
     end
 
-    def display_headers
-      # optionally override this method to display custom headers
+    def title
+      self.class.to_s
     end
 
     def list_attributes

data/lib/superset/request.rb

@@ -5,12 +5,13 @@ module Superset
     class InvalidParameterError < StandardError; end
     class ValidationError < StandardError; end
 
-    PAGE_SIZE = 100
+    DEFAULT_PAGE_SIZE = 100
 
-    attr_accessor :page_num
+    attr_accessor :page_num, :page_size
 
-    def initialize(page_num: 0)
+    def initialize(page_num: 0, page_size: nil)
       @page_num = page_num
+      @page_size = page_size || DEFAULT_PAGE_SIZE
     end
 
     def self.call
@@ -51,7 +52,8 @@ module Superset
     end
 
     def pagination
-      "page:#{page_num},page_size:#{PAGE_SIZE}"
+      raise InvalidParameterError, "page_size max is 1000 records" if page_size.to_i > 1000
+      "page:#{page_num},page_size:#{page_size}"
     end
 
     def filters

data/lib/superset/security/permissions_resources/list.rb

@@ -3,8 +3,8 @@ module Superset
     module PermissionsResources
       class List < Superset::Request
 
-        def initialize(page_num: 0)         
-          super(page_num: page_num)
+        def initialize(**kwargs)
+          super(**kwargs)
         end
 
         private

data/lib/superset/security/user/list.rb

@@ -4,10 +4,10 @@ module Superset
       class List < Superset::Request
         attr_reader :email_contains, :username_equals
 
-        def initialize(page_num: 0, email_contains: '', username_equals: '')
+        def initialize(email_contains: '', username_equals: '', **kwargs)
           @email_contains = email_contains
           @username_equals = username_equals
-          super(page_num: page_num)
+          super(**kwargs)
         end
 
         private

data/lib/superset/services/duplicate_dashboard.rb

@@ -9,15 +9,17 @@ module Superset
   module Services
     class DuplicateDashboard < Superset::Request
 
-      attr_reader :source_dashboard_id, :target_schema, :target_database_id, :allowed_domains, :tags, :publish
+      attr_reader :source_dashboard_id, :target_schema, :target_database_id, :target_dataset_suffix_override, :allowed_domains, :tags, :publish, :target_catalog_name
 
-      def initialize(source_dashboard_id:, target_schema:, target_database_id: , allowed_domains: [], tags: [], publish: false)
+      def initialize(source_dashboard_id:, target_schema:, target_database_id: , target_dataset_suffix_override: nil, allowed_domains: [], tags: [], publish: false, target_catalog_name: nil)
         @source_dashboard_id = source_dashboard_id
         @target_schema = target_schema
         @target_database_id = target_database_id
+        @target_dataset_suffix_override = target_dataset_suffix_override
         @allowed_domains = allowed_domains
         @tags = tags
         @publish = publish
+        @target_catalog_name = target_catalog_name
       end
 
       def perform
@@ -55,6 +57,11 @@ module Superset
 
       rescue => e
         logger.error("#{e.message}")
+        remove_duplicated_objects
+        puts "------------------------------------------------------------------------------\n"
+        puts "DUPLICATE DASHBOARD FAILED - ERROR: #{e.message}\n"
+        puts "REMOVED DUPLICATED OBJECTS - Check log/superset-client.log for more details\n"
+        puts "------------------------------------------------------------------------------\n"
         raise e
       end
 
@@ -67,12 +74,12 @@ module Superset
       def add_tags_to_new_dashboard
         return unless tags.present?
 
-        Superset::Tag::AddToObject.new(object_type_id: ObjectType::DASHBOARD, object_id: new_dashboard.id, tags: tags).perform
+        Superset::Tag::AddToObject.new(object_type_id: ObjectType::DASHBOARD, target_id: new_dashboard.id, tags: tags).perform
         logger.info "  Added tags to dashboard #{new_dashboard.id}: #{tags}"
       rescue => e
         # catching tag error and display in log .. but also alowing the process to finish logs as tag error is fairly insignificant
-        logger.error("  FAILED to add tags to new dashboard id: #{new_dashboard.id}. Error is #{e.message}")
-        logger.error("  Missing Tags Values are #{tags}")
+        logger.error("  FAILED to add tags to new dashboard id: #{new_dashboard.id}. Error: #{e.message}")
+        raise ValidationError, "Failed to add tags to new dashboard id: #{new_dashboard.id}. #{e.message}. Missing Tags Values are #{tags}"
       end
 
       def created_embedded_config
@@ -90,31 +97,39 @@ module Superset
 
       def duplicate_source_dashboard_datasets
         source_dashboard_datasets.each do |dataset|
-          # duplicate the dataset, renaming to use of suffix as the target_schema
-          # reason: there is a bug(or feature) in the SS API where a dataset name must be uniq when duplicating.  
-          # (note however renaming in the GUI to a dup name works fine)
-          new_dataset_name = "#{dataset[:datasource_name]}-#{target_schema}"
-          existing_datasets = Superset::Dataset::List.new(title_equals: new_dataset_name, schema_equals: target_schema).result
+          # duplicate the dataset, renaming to use of suffix as the target_schema OR target_dataset_suffix_override value
+          # reason: there is a bug(or feature) in the SS where a dataset name must be uniq when duplicating
+          target_dataset_name = new_dataset_name(dataset[:datasource_name])
+          existing_datasets = Superset::Dataset::List.new(title_equals: target_dataset_name, schema_equals: target_schema).result
           if existing_datasets.any?
-            logger.info "Dataset #{existing_datasets[0]["table_name"]} already exists. Reusing it"
-            new_dataset_id = existing_datasets[0]["id"] # assuming that we do not name multiple datasets with same name in a single schema
+            logger.info "  Dataset #{target_dataset_name} already exists. Reusing it"
+            new_dataset_id = existing_datasets[0]["id"]
           else
-            new_dataset_id = Superset::Dataset::Duplicate.new(source_dataset_id: dataset[:id], new_dataset_name: new_dataset_name).perform
-            # update the new dataset with the target schema and target database
-            Superset::Dataset::UpdateSchema.new(source_dataset_id: new_dataset_id, target_database_id: target_database_id, target_schema: target_schema).perform
+            new_dataset_id = Superset::Dataset::Duplicate.new(source_dataset_id: dataset[:id], new_dataset_name: target_dataset_name).perform
+            # update the new dataset with the target schema, database, catalog
+            Superset::Dataset::UpdateSchema.new(
+              source_dataset_id:  new_dataset_id,
+              target_database_id: target_database_id,
+              target_schema:      target_schema,
+              target_catalog:     validated_target_database_catalog_name).perform
           end
           # keep track of the previous dataset and the matching new dataset_id
           dataset_duplication_tracker <<  { source_dataset_id: dataset[:id], new_dataset_id: new_dataset_id }
         end
       end
 
+      # if a suffix is provided, use it to suffix the dataset name
+      # if no suffix is provided, use the schema name as the suffix
+      def new_dataset_name(dataset_name)
+        return "#{dataset_name}-#{target_schema}" if target_dataset_suffix_override.blank?
+        "#{dataset_name}-#{target_dataset_suffix_override.downcase}"
+      end
+
       def update_charts_with_new_datasets
         logger.info "Updating Charts to point to New Datasets and updating Dashboard json_metadata ..."
         # note dashboard json_metadata currently still points to the old chart ids and is updated here
-
         new_dashboard_json_metadata_json_string = new_dashboard_json_metadata_configuration.to_json # need to convert to string for gsub
-        # get all chart ids for the new dashboard
-        new_charts_list = Superset::Dashboard::Charts::List.new(new_dashboard.id).result
+
         new_chart_ids_list = new_charts_list&.map { |r| r['id'] }&.compact
         # get all chart details for the source dashboard
         original_charts = Superset::Dashboard::Charts::List.new(source_dashboard_id).result.map { |r| [r['slice_name'], r['id']] }.to_h
@@ -144,6 +159,11 @@ module Superset
         @new_dashboard_json_metadata_configuration = JSON.parse(new_dashboard_json_metadata_json_string)
       end
 
+      def new_charts_list
+        # get all chart ids for the new dashboard
+        @new_charts_list ||= Superset::Dashboard::Charts::List.new(new_dashboard.id).result
+      end
+
       def duplicate_source_dashboard_filters
         return unless source_dashboard_filter_dataset_ids.length.positive?
 
@@ -160,11 +180,11 @@ module Superset
 
       def update_source_dashboard_json_metadata
         logger.info "  Updated new Dashboard json_metadata charts with new dataset ids"
-        Superset::Dashboard::Put.new(target_dashboard_id: new_dashboard.id, params: { "json_metadata" => @new_dashboard_json_metadata_configuration.to_json }).perform
+        Superset::Dashboard::Put.new(target_id: new_dashboard.id, params: { "json_metadata" => @new_dashboard_json_metadata_configuration.to_json }).perform
       end
 
       def publish_dashboard
-        Superset::Dashboard::Put.new(target_dashboard_id: new_dashboard.id, params: { published: publish } ).perform
+        Superset::Dashboard::Put.new(target_id: new_dashboard.id, params: { published: publish } ).perform
       end
 
       def new_dashboard
@@ -205,7 +225,7 @@ module Superset
         raise ValidationError, "One or more source dashboard filters point to a different schema than the dashboard charts. Identified Unpermittied Filter Dataset Ids are #{unpermitted_filter_dataset_ids.to_s}" if unpermitted_filter_dataset_ids.any?
 
         # new dataset validations - Need to be commented for EU dashboard duplication as we are using the existing datasets for the new dashboard
-        raise ValidationError, "DATASET NAME CONFLICT: The Target Schema #{target_schema} already has existing datasets named: #{target_schema_matching_dataset_names.join(',')}" unless target_schema_matching_dataset_names.empty?
+        raise ValidationError, "DATASET NAME CONFLICT: The Target Database #{target_database_id} with Schema #{target_schema} already has existing datasets named: #{target_schema_matching_dataset_names.join(',')}" unless target_schema_matching_dataset_names.empty?
         validate_source_dashboard_datasets_sql_does_not_hard_code_schema
 
         # embedded allowed_domain validations
@@ -214,8 +234,8 @@ module Superset
 
       def validate_source_dashboard_datasets_sql_does_not_hard_code_schema
         errors = source_dashboard_datasets.map do |dataset|
-          "The Dataset ID #{dataset[:id]} SQL query is hard coded with the schema value and can not be duplicated cleanly.  " +
-            "Remove all direct embedded schema calls from the Dataset SQL query before continuing." if dataset[:sql].include?("#{dataset[:schema]}.")
+          "The Dataset ID #{dataset[:id]} SQL query is hard coded with the schema value: #{dataset[:schema]}. This indicates that the dataset can not be duplicated cleanly to point to the target schema.  " +
+            "Remove all direct embedded schema calls from the Dataset SQL query before continuing." if dataset[:sql]&.include?("#{dataset[:schema]}.")
         end.compact
         raise ValidationError, errors.join("\n") unless errors.empty?
       end
@@ -241,13 +261,15 @@ module Superset
         source_dashboard_datasets.map { |dataset| dataset[:datasource_name] }.uniq
       end
 
-      # identify any already existing datasets in the target schema that have the same name as the source dashboard datasets
+      # identify any already existing datasets in the target database and schema that have the same name as the source dashboard datasets + suffix
       # note this is prior to adding the (COPY) suffix
-      # here we will need to decide if we want to use the existing dataset or not see NEP-????
+      # here we will need to decide if we want to use the existing dataset or not
       # for now we will exit with an error if we find any existing datasets of the same name
       def target_schema_matching_dataset_names
         @target_schema_matching_dataset_names ||= source_dashboard_dataset_names.map do |source_dataset_name|
-          existing_names = Superset::Dataset::List.new(title_contains: source_dataset_name, schema_equals: target_schema).result.map{|t|t['table_name']}.uniq # contains match to cover with suffix as well
+          source_dataset_name_with_suffix = new_dataset_name(source_dataset_name)
+
+          existing_names = Superset::Dataset::List.new(title_contains: source_dataset_name_with_suffix, database_id_eq: target_database_id, schema_equals: target_schema).result.map{|t|t['table_name']}.uniq # contains match to cover with suffix as well
           unless existing_names.flatten.empty?
             logger.error "  HALTING PROCESS: Schema #{target_schema} already has Dataset called #{existing_names}"
           end
@@ -263,6 +285,15 @@ module Superset
         @filter_dataset_ids ||= source_dashboard.filter_configuration.map { |c| c['targets'] }.flatten.compact.map { |c| c['datasetId'] }.flatten.compact.uniq
       end
 
+      # if multiple catalogs are present, the target_catalog_name must be provided, otherwise use the first catalog
+      def validated_target_database_catalog_name
+        catalogs = Superset::Database::GetCatalogs.new(target_database_id).catalogs
+        return target_catalog_name if catalogs.include?(target_catalog_name)
+
+        raise ValidationError, "Target Database #{target_database_id} has multiple catalogs, must provide target_catalog_name" if catalogs.size > 1 && target_catalog_name.blank?
+        @validated_target_database_catalog_name ||= catalogs.find { |c| c['name'] == target_catalog_name } || catalogs.first
+      end
+
       # Primary Assumption is that all charts datasets on the source dashboard are pointing to the same database schema
       # An unpermitted filter will have a dataset that is pulling data from a datasource that is
       # different to the dashboard charts database schema
@@ -279,6 +310,21 @@ module Superset
         end
       end
 
+      # remove the confirmed duplicated objects if the process fails
+      # do not use Dashboard::BulkDeleteCascade here as it may remove datasets from the source dashboard as well
+      def remove_duplicated_objects
+        logger.info "Removing duplicated objects ..."
+
+        new_dataset_ids = dataset_duplication_tracker&.map { |dataset| dataset[:new_dataset_id] }.compact
+        Superset::Dataset::BulkDelete.new(dataset_ids: new_dataset_ids).perform if new_dataset_ids.any?
+
+        new_chart_ids = new_charts_list&.map { |r| r['id'] }.compact
+        Superset::Chart::BulkDelete.new(chart_ids: new_chart_ids).perform if new_chart_ids.any?
+
+        Superset::Dashboard::Delete.new(dashboard_id: new_dashboard.id).perform if new_dashboard.id.present?
+        logger.info "Removed duplicated objects successfully."
+      end
+
       def logger
         @logger ||= Superset::Logger.new
       end

data/lib/superset/services/import_dashboard_across_environment.rb

@@ -56,6 +56,7 @@ module Superset
 
       def remove_source_database_config
         return if dashboard_config[:databases].blank?
+
         previous_database_name = dashboard_config[:databases]&.first[:content][:database_name]
         File.delete(dashboard_config[:databases].first[:filename])
 
@@ -78,6 +79,8 @@ module Superset
         dashboard_config[:datasets].each do |dataset|
           dataset[:content][:database_uuid] = dashboard_config[:databases].first[:content][:uuid]
           dataset[:content][:schema]        = target_database_schema
+          dataset[:content][:catalog]       = nil # reset target to use default catalog if applicable
+
           stringified_content = deep_transform_keys_to_strings(dataset[:content])
           File.open(dataset[:filename], 'w') { |f| f.write stringified_content.to_yaml }
         end

data/lib/superset/tag/add_to_object.rb

@@ -4,11 +4,11 @@ module Superset
   module Tag
     class AddToObject < Superset::Request
 
-      attr_reader :object_type_id, :object_id, :tags
+      attr_reader :object_type_id, :target_id, :tags
 
-      def initialize(object_type_id:, object_id:, tags: [])
+      def initialize(object_type_id:, target_id:, tags: [])
         @object_type_id = object_type_id
-        @object_id = object_id
+        @target_id = target_id
         @tags = tags
       end
 
@@ -31,7 +31,7 @@ module Superset
       def validate_constructor_args
         raise InvalidParameterError, "object_type_id integer is required" unless object_type_id.present? && object_type_id.is_a?(Integer)
         raise InvalidParameterError, "object_type_id is not a known value" unless ObjectType.list.include?(object_type_id)
-        raise InvalidParameterError, "object_id integer is required" unless object_id.present? && object_id.is_a?(Integer)
+        raise InvalidParameterError, "target_id integer is required" unless target_id.present? && target_id.is_a?(Integer)
         raise InvalidParameterError, "tags array is required" unless tags.present? && tags.is_a?(Array)
         raise InvalidParameterError, "tags array must contain string only values" unless tags.all? { |item| item.is_a?(String) }
       end
@@ -39,7 +39,7 @@ module Superset
       private
 
       def route
-        "tag/#{object_type_id}/#{object_id}/"
+        "tag/#{object_type_id}/#{target_id}/"
       end
     end
   end

data/lib/superset/tag/list.rb

@@ -23,7 +23,7 @@ module Superset
         # TODO filtering across all list classes can be refactored to support multiple options in a more flexible way
         filter_set = []
         filter_set << "(col:name,opr:ct,value:'#{name_contains}')" if name_contains.present?
-        filter_set << "(col:name,opr:eq,value:#{name_equals})" if name_equals.present?  
+        filter_set << "(col:name,opr:eq,value:'#{name_equals}')" if name_equals.present?
         unless filter_set.empty?
           "filters:!(" + filter_set.join(',') + "),"
         end

data/lib/superset/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Superset
-  VERSION = "0.2.6"
+  VERSION = "0.3.0"
 end

data/superset.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.summary = "A Ruby Client for Apache Superset API"
   spec.homepage = "https://github.com/rdytech/superset-client"
   spec.license = "MIT"
-  spec.required_ruby_version = ">= 2.7.8"
+  spec.required_ruby_version = ">= 3"
 
   #spec.metadata["allowed_push_host"] = ""
 
@@ -34,21 +34,20 @@ Gem::Specification.new do |spec|
     "lib"
   ]
 
-  # Uncomment to register a new dependency of your gem
-  spec.add_dependency "dotenv", "~> 2.7"
-  spec.add_dependency "json", "~> 2.6"
+  spec.add_dependency "json", ">= 2.0"
   spec.add_dependency "terminal-table", "~> 4.0"
-  spec.add_dependency "rake", "~> 13.0"
-  spec.add_dependency "rollbar", "~> 3.4"
-  spec.add_dependency "require_all", "~> 3.0"
-  spec.add_dependency "rubyzip", "~> 1.0"
+  spec.add_dependency "require_all", ">= 3.0"
+  spec.add_dependency "rubyzip", ">= 1.3"
   spec.add_dependency "faraday", "~> 1.0"
   spec.add_dependency "faraday-multipart", "~> 1.0"
-  spec.add_dependency "enumerate_it", "~> 1.7.0"
-
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "rubocop", "~> 1.5"
-  spec.add_development_dependency "pry", "~> 0.14"
+  spec.add_dependency "enumerate_it", ">= 1.7"
+
+  spec.add_development_dependency "dotenv", ">= 2.0"
+  spec.add_development_dependency "rake", ">= 13.0"
+  spec.add_development_dependency "rspec", ">= 3.0"
+  spec.add_development_dependency "rubocop", ">= 1.0"
+  spec.add_development_dependency "pry", ">= 0.14"
+  spec.add_development_dependency "rollbar", ">= 3.0"
 
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html