superset 0.1.6

data/lib/superset/security/user/get.rb

@@ -0,0 +1,27 @@
+module Superset
+  module Security
+    module User
+      class Get < Superset::Request
+        attr_reader :id
+
+        def initialize(id)
+          @id = id
+        end
+
+        def result
+          [ super ]
+        end
+
+        private
+
+        def list_attributes
+          [:id, :first_name, :last_name, :email, :login_count, :last_login]
+        end
+
+        def route
+          "security/users/#{id}"
+        end
+      end
+    end
+  end
+end

data/lib/superset/security/user/list.rb

@@ -0,0 +1,42 @@
+module Superset
+  module Security
+    module User
+      class List < Superset::Request
+        attr_reader :email_contains, :username_equals
+
+        def initialize(page_num: 0, email_contains: '', username_equals: '')
+          @email_contains = email_contains
+          @username_equals = username_equals
+          super(page_num: page_num)
+        end
+
+        private
+
+        def route
+          "security/users/?q=(#{query_params})"
+        end
+
+        def filters
+          raise 'ERROR: only one filter supported currently' if  email_contains.present? && username_equals.present?
+
+          if email_contains.present?
+            "filters:!((col:email,opr:ct,value:#{email_contains})),"
+          elsif username_equals.present?
+            "filters:!((col:username,opr:eq,value:#{username_equals})),"
+          else
+            ''
+          end
+        end
+
+        def title
+          "#{response[:count]} Matching Users for Host: #{superset_host}\n" \
+            "#{result.count} Users listed with: #{query_params}"
+        end
+
+        def list_attributes
+          [:id, :first_name, :last_name, :email, :active, :login_count, :last_login]
+        end
+      end
+    end
+  end
+end

data/lib/superset/services/duplicate_dashboard.rb

@@ -0,0 +1,298 @@
+# Data Sovereignty validations are enforced, ie confirming
+# - all charts datasets on the source dashboard are pointing to the same database schema
+# - all filter datasets on the source dashboard are pointing to the same database schema as the charts
+
+# - The source dashboard is in the same Superset instance as the target database and target schema
+
+
+module Superset
+  module Services
+    class DuplicateDashboard < Superset::Request
+
+      attr_reader :source_dashboard_id, :target_schema, :target_database_id, :allowed_domains, :tags, :publish
+
+      def initialize(source_dashboard_id:, target_schema:, target_database_id: , allowed_domains: [], tags: [], publish: false)
+        @source_dashboard_id = source_dashboard_id
+        @target_schema = target_schema
+        @target_database_id = target_database_id
+        @allowed_domains = allowed_domains
+        @tags = tags
+        @publish = publish
+      end
+
+      def perform
+        # validate all params before starting the process
+        validate_params
+
+        # Pull the Datasets for all charts on the source dashboard
+        source_dashboard_datasets
+
+        # create a new_dashboard by copying the source_dashboard using with 'duplicate_slices: true' to get a new set of charts.
+        # The new_dashboard will have a copy of charts from the source_dashboard, but with the same datasets as the source_dashboard
+        new_dashboard
+
+        # Duplicate these Datasets to the new target schema and target database
+        duplicate_source_dashboard_datasets
+
+        # Update the Charts on the New Dashboard with the New Datasets and update the Dashboard json_metadata for the charts
+        update_charts_with_new_datasets
+
+        # Duplicate filters to the new target schema and target database
+        duplicate_source_dashboard_filters
+
+        update_source_dashboard_json_metadata
+
+        created_embedded_config
+
+        add_tags_to_new_dashboard
+
+        publish_dashboard if publish
+
+        end_log_message
+
+        # return the new dashboard id and url
+        { new_dashboard_id: new_dashboard.id, new_dashboard_url: new_dashboard.url, published: publish }
+
+      rescue => e
+        logger.error("#{e.message}")
+        raise e
+      end
+
+      def new_dashboard_json_metadata_configuration
+        @new_dashboard_json_metadata_configuration ||= new_dashboard.json_metadata
+      end
+
+      private
+
+      def add_tags_to_new_dashboard
+        return unless tags.present?
+
+        Superset::Tag::AddToObject.new(object_type_id: ObjectType::DASHBOARD, object_id: new_dashboard.id, tags: tags).perform
+        logger.info "  Added tags to dashboard #{new_dashboard.id}: #{tags}"
+      rescue => e
+        # catching tag error and display in log .. but also alowing the process to finish logs as tag error is fairly insignificant
+        logger.error("  FAILED to add tags to new dashboard id: #{new_dashboard.id}. Error is #{e.message}")
+        logger.error("  Missing Tags Values are #{tags}")
+      end
+
+      def created_embedded_config
+        return unless allowed_domains.present?
+
+        result = Dashboard::Embedded::Put.new(dashboard_id: new_dashboard.id, allowed_domains: allowed_domains).result
+        logger.info "Added Embedded Settings to New Dashboard #{new_dashboard.id}:"
+        logger.info "  Embedded Domain allowed_domains: #{result['allowed_domains']}"
+        logger.info "  Embedded UUID: #{result['uuid']}"
+      end
+
+      def dataset_duplication_tracker
+        @dataset_duplication_tracker ||= []
+      end
+
+      def duplicate_source_dashboard_datasets
+        source_dashboard_datasets.each do |dataset|
+          # duplicate the dataset, renaming to use of suffix as the target_schema
+          # reason: there is a bug(or feature) in the SS API where a dataset name must be uniq when duplicating.  
+          # (note however renaming in the GUI to a dup name works fine)
+          new_dataset_id = Superset::Dataset::Duplicate.new(source_dataset_id: dataset[:id], new_dataset_name: "#{dataset[:datasource_name]}-#{target_schema}").perform
+
+          # keep track of the previous dataset and the matching new dataset_id
+          dataset_duplication_tracker <<  { source_dataset_id: dataset[:id], new_dataset_id: new_dataset_id }
+
+          # update the new dataset with the target schema and target database
+          Superset::Dataset::UpdateSchema.new(source_dataset_id: new_dataset_id, target_database_id: target_database_id, target_schema: target_schema).perform
+        end
+      end
+
+      def update_charts_with_new_datasets
+        logger.info "Updating Charts to point to New Datasets and updating Dashboard json_metadata ..."
+        # note dashboard json_metadata currently still points to the old chart ids and is updated here
+
+        new_dashboard_json_metadata_json_string = new_dashboard_json_metadata_configuration.to_json # need to convert to string for gsub
+        # get all chart ids for the new dashboard
+        new_charts_list = Superset::Dashboard::Charts::List.new(new_dashboard.id).result
+        new_chart_ids_list = new_charts_list&.map { |r| r['id'] }&.compact
+        # get all chart details for the source dashboard
+        original_charts = Superset::Dashboard::Charts::List.new(source_dashboard_id).result.map { |r| [r['slice_name'], r['id']] }.to_h
+        new_charts = new_charts_list.map { |r| [r['id'], r['slice_name']] }.to_h
+        return unless new_chart_ids_list.any?
+
+        # for each chart, update the charts current dataset_id with the new dataset_id
+        new_chart_ids_list.each do |new_chart_id|
+
+          # get the CURRENT dataset_id for the new chart
+          current_chart_dataset_id = Superset::Chart::Get.new(new_chart_id).datasource_id
+
+          # find the new dataset_id for the new chart, based on the current_chart_dataset_id
+          new_dataset_id = dataset_duplication_tracker.find { |dataset| dataset[:source_dataset_id] == current_chart_dataset_id }&.fetch(:new_dataset_id, nil)
+
+          # update the new chart to target the new dataset_id and to the reference the new target_dashboard_id
+          Superset::Chart::UpdateDataset.new(chart_id: new_chart_id, target_dataset_id: new_dataset_id, target_dashboard_id: new_dashboard.id).perform
+          logger.info "  Update Chart #{new_chart_id} to new dataset_id #{new_dataset_id}"
+
+          # update json metadata swaping the old chart_id with the new chart_id
+          original_chart_id = original_charts[new_charts[new_chart_id]]
+          regex_with_numeric_boundaries = Regexp.new("\\b#{original_chart_id.to_s}\\b")
+          new_dashboard_json_metadata_json_string.gsub!(regex_with_numeric_boundaries, new_chart_id.to_s)
+        end
+
+        # convert back to hash .. and store in the new_dashboard_json_metadata_configuration
+        @new_dashboard_json_metadata_configuration = JSON.parse(new_dashboard_json_metadata_json_string)
+      end
+
+      def duplicate_source_dashboard_filters
+        return unless source_dashboard_filter_dataset_ids.length.positive?
+
+        logger.info "Updating Filters to point to new dataset targets ..."
+        configuration = new_dashboard_json_metadata_configuration['native_filter_configuration']&.map do |filter_config|
+          targets = filter_config['targets']
+          target_filter_dataset_id = dataset_duplication_tracker.find { |d| d[:source_dataset_id] == targets.first["datasetId"] }&.fetch(:new_dataset_id, nil)
+          filter_config['targets'] = [targets.first.merge({ "datasetId"=> target_filter_dataset_id })]
+          filter_config
+        end
+
+        @new_dashboard_json_metadata_configuration['native_filter_configuration'] = configuration || []
+      end
+
+      def update_source_dashboard_json_metadata
+        logger.info "  Updated new Dashboard json_metadata charts with new dataset ids"
+        Superset::Dashboard::Put.new(target_dashboard_id: new_dashboard.id, params: { "json_metadata" => @new_dashboard_json_metadata_configuration.to_json }).perform
+      end
+
+      def publish_dashboard
+        Superset::Dashboard::Put.new(target_dashboard_id: new_dashboard.id, params: { published: publish } ).perform
+      end
+
+      def new_dashboard
+        @new_dashboard ||= begin
+          copy = Superset::Dashboard::Copy.new(
+            source_dashboard_id:       source_dashboard_id,
+            duplicate_slices:          true,
+            clear_shared_label_colors: true
+          ).perform
+          logger.info("  Copy Dashboard/Charts Completed - New Dashboard ID: #{copy.id}")
+          copy
+        end
+      rescue => e
+        logger.info("  Dashboard::Copy error: #{e.message}")
+        raise "Dashboard::Copy error: #{e.message}"
+      end
+
+      # retrieve the datasets that will be duplicated
+      def source_dashboard_datasets
+        @source_dashboard_datasets ||= Superset::Dashboard::Datasets::List.new(source_dashboard_id).datasets_details
+      rescue => e
+        raise "Unable to retrieve datasets for source dashboard #{source_dashboard_id}: #{e.message}"
+      end
+
+      def validate_params
+        start_log_msg
+        # params validations
+        raise  InvalidParameterError, "source_dashboard_id integer is required" unless source_dashboard_id.present? && source_dashboard_id.is_a?(Integer)
+        raise  InvalidParameterError, "target_schema string is required" unless target_schema.present? && target_schema.is_a?(String)
+        raise  InvalidParameterError, "target_database_id integer is required" unless target_database_id.present? && target_database_id.is_a?(Integer)
+
+        # dashboard validations
+        # Validation of source dashboard existance will occur inside the new_dashboard call
+
+        # schema validations
+        raise ValidationError, "Schema #{target_schema} does not exist in target database: #{target_database_id}" unless target_database_available_schemas.include?(target_schema)
+        raise ValidationError, "The source dashboard datasets are required to point to one schema only. Actual schema list is #{source_dashboard_schemas.join(',')}" if source_dashboard_has_more_than_one_schema?
+        raise ValidationError, "One or more source dashboard filters point to a different schema than the dashboard charts. Identified Unpermittied Filter Dataset Ids are #{unpermitted_filter_dataset_ids.to_s}" if unpermitted_filter_dataset_ids.any?
+
+        # new dataset validations
+        raise ValidationError, "DATASET NAME CONFLICT: The Target Schema #{target_schema} already has existing datasets named: #{target_schema_matching_dataset_names.join(',')}" unless target_schema_matching_dataset_names.empty?
+        validate_source_dashboard_datasets_sql_does_not_hard_code_schema
+
+        # embedded allowed_domain validations
+        raise  InvalidParameterError, 'allowed_domains array is required' if allowed_domains.nil? || allowed_domains.class != Array
+      end
+
+      def validate_source_dashboard_datasets_sql_does_not_hard_code_schema
+        errors = source_dashboard_datasets.map do |dataset|
+          "The Dataset ID #{dataset[:id]} SQL query is hard coded with the schema value and can not be duplicated cleanly.  " +
+            "Remove all direct embedded schema calls from the Dataset SQL query before continuing." if dataset[:sql].include?("#{dataset[:schema]}.")
+        end.compact
+        raise ValidationError, errors.join("\n") unless errors.empty?
+      end
+
+      def source_dashboard
+        @source_dashboard ||= Superset::Dashboard::Get.new(source_dashboard_id)
+      end
+
+      def target_database_available_schemas
+        Superset::Database::GetSchemas.call(target_database_id)
+      end
+
+      def source_dashboard_has_more_than_one_schema?
+        source_dashboard_schemas.count > 1
+      end
+
+      # Data Sovereignty rules expect only 1 value here, and raise a validation error if there is > 1
+      def source_dashboard_schemas
+        source_dashboard_datasets.map { |dataset| dataset[:schema] }.uniq
+      end
+
+      def source_dashboard_dataset_names
+        source_dashboard_datasets.map { |dataset| dataset[:datasource_name] }.uniq
+      end
+
+      # identify any already existing datasets in the target schema that have the same name as the source dashboard datasets
+      # note this is prior to adding the (COPY) suffix
+      # here we will need to decide if we want to use the existing dataset or not see NEP-????
+      # for now we will exit with an error if we find any existing datasets of the same name
+      def target_schema_matching_dataset_names
+        source_dashboard_dataset_names.map do |source_dataset_name|
+          existing_names = Superset::Dataset::List.new(title_contains: source_dataset_name, schema_equals: target_schema).result.map{|t|t['table_name']}.uniq # contains match to cover with suffix as well
+          unless existing_names.flatten.empty?
+            logger.error "  HALTING PROCESS: Schema #{target_schema} already has Dataset called #{existing_names}"
+          end
+          existing_names
+        end.flatten.compact
+      end
+
+      def source_dashboard_dataset_ids
+        source_dashboard_datasets.map{|d|d['id']}
+      end
+
+      def source_dashboard_filter_dataset_ids
+        filters_configuration = JSON.parse(source_dashboard.result['json_metadata'])['native_filter_configuration'] || []
+        return Array.new unless filters_configuration && filters_configuration.any?
+
+        # pull only the filters dataset ids from the dashboard
+        filters_configuration.map { |c| c['targets'] }.flatten.compact.map { |c| c['datasetId'] }.flatten.compact
+      end
+
+      # Primary Assumption is that all charts datasets on the source dashboard are pointing to the same database schema
+      # An unpermitted filter will have a dataset that is pulling data from a datasource that is
+      # different to the dashboard charts database schema
+      def unpermitted_filter_dataset_ids
+        @unpermitted_filter_dataset_ids ||= begin
+          filter_datasets_not_used_in_charts = source_dashboard_filter_dataset_ids - source_dashboard_dataset_ids
+
+          # retrieve any filter_datasets_not_used_in_charts that do not match the source_dashboard_schema
+          filter_datasets_not_used_in_charts.map do |filter_dataset|
+            filter_dataset_schema = Superset::Dataset::Get.new(filter_dataset).schema
+            # return any filter datasets not used in charts that are from a different schema
+            {  filter_dataset_id: filter_dataset, filter_schema: filter_dataset_schema  } if [filter_dataset_schema] != source_dashboard_schemas
+          end.compact
+        end
+      end
+
+      def logger
+        @logger ||= Superset::Logger.new
+      end
+
+      def start_log_msg
+        logger.info ""
+        logger.info ">>>>>>>>>>>>>>>>> Starting DuplicateDashboard Service <<<<<<<<<<<<<<<<<<<<<<"
+        logger.info "Source Dashboard URL: #{source_dashboard.url}"
+        logger.info "Duplicating dashboard #{source_dashboard_id} into Target Schema: #{target_schema} in database #{target_database_id}"
+      end
+
+      def end_log_message
+        logger.info "Duplication Successful. New Dashboard URL: #{new_dashboard.url} "
+        logger.info ">>>>>>>>>>>>>>>>> Finished DuplicateDashboard Service <<<<<<<<<<<<<<<<<<<<<<"
+      end
+    end
+  end
+end

data/lib/superset/sqllab/execute.rb

@@ -0,0 +1,52 @@
+module Superset
+  module Sqllab
+    class Execute < Superset::Request
+      class InvalidParameterError < StandardError; end
+
+      attr_reader :database_id, :query, :schema, :query_limit
+
+      def initialize(database_id: , query: , schema: 'public', query_limit: 1000)
+        @database_id = database_id
+        @query = query
+        @schema = schema
+        @query_limit = query_limit
+      end
+
+      def perform
+        validate_constructor_args
+        response
+        data
+      end
+
+      def response
+        @response ||= client.post(route, query_params)
+      end
+
+      def data
+        response["data"]
+      end
+
+      private
+
+      def route
+        "sqllab/execute/"
+      end
+
+      def query_params
+        {
+          database_id: database_id,
+          sql:         query,
+          schema:      schema,
+          queryLimit:  query_limit,
+          runAsync: false,
+        }
+      end
+
+      def validate_constructor_args
+        raise InvalidParameterError, "database_id integer is required" unless database_id.present? && database_id.is_a?(Integer)
+        raise InvalidParameterError, "query string is required" unless query.present? && query.is_a?(String)
+        raise InvalidParameterError, "schema must be a String type" unless schema.is_a?(String)
+      end
+    end
+  end
+end

data/lib/superset/tag/add_to_object.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Superset
+  module Tag
+    class AddToObject < Superset::Request
+
+      attr_reader :object_type_id, :object_id, :tags
+
+      def initialize(object_type_id:, object_id:, tags: [])
+        @object_type_id = object_type_id
+        @object_id = object_id
+        @tags = tags
+      end
+
+      def perform
+        validate_constructor_args
+
+        response  # NOTE API response for success is {} .. not particularly informative
+      end
+
+      def response
+        @response ||= client.post(route, params)
+      end
+
+      def params
+        {
+          "properties": { "tags": tags }
+        }
+      end
+
+      def validate_constructor_args
+        raise InvalidParameterError, "object_type_id integer is required" unless object_type_id.present? && object_type_id.is_a?(Integer)
+        raise InvalidParameterError, "object_type_id is not a known value" unless ObjectType.list.include?(object_type_id)
+        raise InvalidParameterError, "object_id integer is required" unless object_id.present? && object_id.is_a?(Integer)
+        raise InvalidParameterError, "tags array is required" unless tags.present? && tags.is_a?(Array)
+        raise InvalidParameterError, "tags array must contain string only values" unless tags.all? { |item| item.is_a?(String) }
+      end
+
+      private
+
+      def route
+        "tag/#{object_type_id}/#{object_id}/"
+      end
+    end
+  end
+end

data/lib/superset/tag/get.rb

@@ -0,0 +1,30 @@
+module Superset
+  module Tag
+    class Get < Superset::Request
+
+      attr_reader :id
+
+      def initialize(id)
+        @id = id
+      end
+
+      def self.call(id)
+        self.new(id).list
+      end
+
+      def result
+        [ super ]
+     end
+
+      private
+
+      def list_attributes
+        ['id', 'name', 'type', 'description']
+      end
+
+      def route
+        "tag/#{id}"
+      end
+    end
+  end
+end

data/lib/superset/tag/list.rb

@@ -0,0 +1,37 @@
+module Superset
+  module Tag
+    class List < Superset::Request
+      attr_reader :name_contains, :name_equals
+
+      def initialize(page_num: 0, name_contains: '', name_equals: '')
+        @name_contains = name_contains
+        @name_equals = name_equals
+        super(page_num: page_num)
+      end
+
+      def self.call
+        self.new.list
+      end
+
+      private
+
+      def route
+        "tag/?q=(#{query_params})"
+      end
+
+      def filters
+        # TODO filtering across all list classes can be refactored to support multiple options in a more flexible way
+        filter_set = []
+        filter_set << "(col:name,opr:ct,value:'#{name_contains}')" if name_contains.present?
+        filter_set << "(col:name,opr:eq,value:#{name_equals})" if name_equals.present?  
+        unless filter_set.empty?
+          "filters:!(" + filter_set.join(',') + "),"
+        end
+      end
+
+      def list_attributes
+        ['id', 'name', 'description']
+      end
+    end
+  end
+end

data/lib/superset/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Superset
+  VERSION = "0.1.6"
+end

data/lib/superset.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'require_all'
+
+require_rel "superset/credential"
+require_relative "superset/authenticator"
+require_relative "superset/client"
+require_relative "superset/display"
+require_relative "superset/logger"
+require_relative "superset/request"
+
+require_rel "superset"
+
+module Superset
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/log/README.md

@@ -0,0 +1,4 @@
+Log outputs of specific superset scription processes to be sent here.
+
+Dashboard Duplication Logs current logged to
+- log/superset-client.log

data/superset.gemspec

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative "lib/superset/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "superset"
+  spec.version = Superset::VERSION
+  spec.authors = ["jbat"]
+  spec.email = ["jonathon.batson@gmail.com"]
+
+  spec.summary = "A Ruby Client for Apache Superset API"
+  spec.homepage = "https://github.com/rdytech/superset-client"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  #spec.metadata["allowed_push_host"] = ""
+
+  #spec.metadata["homepage_uri"] = spec.homepage
+  #spec.metadata["source_code_uri"] = ""
+  #spec.metadata["changelog_uri"] = ""
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git appveyor Gemfile])
+    end
+  end
+
+  #spec.bindir = "exe"
+  #spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = [
+    "lib"
+  ]
+
+  # Uncomment to register a new dependency of your gem
+  spec.add_dependency "dotenv", "~> 2.7"
+  spec.add_dependency "json", "~> 2.6"
+  spec.add_dependency "terminal-table", "~> 1.8"
+  spec.add_dependency "rake", "~> 13.0"
+  spec.add_dependency "rollbar", "~> 3.4"
+  spec.add_dependency "require_all", "~> 3.0"
+  spec.add_dependency "rubyzip", "~> 1.0"
+  spec.add_dependency "faraday", "~> 1.0"
+  spec.add_dependency "faraday-multipart", "~> 1.0"
+  spec.add_dependency "enumerate_it", "~> 1.7.0"
+
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rubocop", "~> 1.5"
+  spec.add_development_dependency "pry", "~> 0.14"
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end