superset 0.1.6 → 0.2.4

data/doc/usage.md

@@ -0,0 +1,105 @@
+# Usage
+
+## API call examples
+
+Quickstart examples
+
+```ruby
+Superset::Database::List.call
+Superset::Database::GetSchemas.new(1).list # get schemas for database 1
+
+Superset::Dashboard::List.call
+Superset::Dashboard::List.new(title_contains: 'Sales').list
+Superset::Dashboard::Datasets::List.new(dashboard_id: 10).list  # get all datasets for dashboard 10
+
+Superset::Dashboard::Datasets::List.new(dashboard_id: 10, include_filter_datasets: true).list  # get all datasets for dashboard 10 including the ones used only in dashboard filters
+Superset::Dashboard::WarmUpCache.new(dashboard_id: 10).perform
+
+Superset::Dataset::List.call
+
+Superset::Chart::List.call
+
+Superset::Dashboard::BulkDelete.new(dashboard_ids: [1,2,3]).perform        # Deletes Dashboards only ( leaves all charts, datasets in place)
+Superset::Dashboard::BulkDeleteCascade.new(dashboard_ids: [1,2,3]).perform # Deletes Dashboards and related charts and datasets. (WARNING: no checks are performed)
+
+Superset::Sqllab::Execute.new(database_id: 1, schema: 'public', query: 'select count(*) from birth_names').perform
+
+Superset::Dashboard::Export.new(dashboard_id: 1, destination_path: '/tmp').perform
+
+Superset::RouteInfo.new(route: 'dashboard/_info').perform # Get info on an API endpoint .. handy for getting available filters
+Superset::RouteInfo.new(route: 'chart/_info').filters     # OR just get the filters for an endpoint
+
+superset_class_list # helper method to list all classes under Superset::
+
+```
+
+## Detailed API Examples
+
+Generally classes follow the convention/path of the Superset API strucuture as per the swagger docs.
+
+ref https://superset.apache.org/docs/api/
+
+Limited support for filters is available on some list pages, primarily through param `title_contains`.  
+Pagination is supported via `page_num` param.
+
+Primary methods across majority of api calls are
+- response : the full API response
+- result : just the result attribute array
+- list : displays a formatted output to console, handy for quick investigation of objects
+- call : is a class method to list on Get and List requests
+
+```ruby
+# List all Databases
+Superset::Database::List.call
+# DEBUG -- : Happi: GET https://your-superset-host/api/v1/database/?q=(page:0,page_size:100), {}
++----+------------------------------------+------------+------------------+
+|                        Superset::Database::List                         |
++----+------------------------------------+------------+------------------+
+| Id | Database name                      | Backend    | Expose in sqllab |
++----+------------------------------------+------------+------------------+
+| 1  | examples                           | postgresql | true             |
++----+------------------------------------+------------+------------------+
+
+# List database schemas for Database 1
+Superset::Database::GetSchemas.new(1).list
+# DEBUG -- : Happi: GET https://your-superset-host/api/v1/database/1/schemas/, {}
+=> ["information_schema", "public"]
+
+# List dashboards
+Superset::Dashboard::List.call
+# PAGE_SIZE is set to 100, so get the second set of 100 dashboards with
+Superset::Dashboard::List.new(page_num: 1).list
+# OR filter by title
+Superset::Dashboard::List.new(title_contains: 'Sales').list
+# DEBUG -- : Happi: GET https://your-superset-host/api/v1/dashboard/?q=(filters:!((col:dashboard_title,opr:ct,value:'Sales')),page:0,page_size:100), {}
+
++-----+------------------------------+-----------+--------------------------------------------------------------------+
+|                                              Superset::Dashboard::List                                              |
++-----+------------------------------+-----------+--------------------------------------------------------------------+
+| Id  | Dashboard title              | Status    | Url                                                                |
++-----+------------------------------+-----------+--------------------------------------------------------------------+
+| 6   | Video Game Sales             | published | https://your-superset-host/superset/dashboard/6/                   |
+| 8   | Sales Dashboard              | published | https://your-superset-host/superset/dashboard/8/                   |
++-----+------------------------------+-----------+--------------------------------------------------------------------+
+
+
+Superset::Dashboard::Get.call(1)  # same as Superset::Dashboard::Get.new(1).list
++----------------------------+
+|     World Banks Data      |
++----------------------------+
+| Charts                     |
++----------------------------+
+| % Rural                    |
+| Region Filter              |
+| Life Expectancy VS Rural % |
+| Box plot                   |
+| Most Populated Countries   |
+| Worlds Population          |
+| Worlds Pop Growth          |
+| Rural Breakdown            |
+| Treemap                    |
+| Growth Rate                |
++----------------------------+
+
+
+```

data/env.sample

@@ -1,4 +1,4 @@
-SUPERSET_HOST="https://your-superset-host.com"
+SUPERSET_HOST="http://localhost:8080/"
 
 # general api calls
 SUPERSET_API_USERNAME="USERNAME"

data/lib/superset/base_put_request.rb

@@ -0,0 +1,30 @@
+module Superset
+  class BasePutRequest < Superset::Request
+    attr_reader :object_id, :params
+
+    def initialize(object_id: ,params: )
+      @object_id = object_id
+      @params = params
+    end
+
+    def perform
+      validate
+      response
+    end
+
+    def response
+      @response ||= client.put(route, params)
+    end
+
+    private
+
+    def validate
+      raise "Error: object_id integer is required" unless object_id.present? && object_id.is_a?(Integer)
+      raise "Error: params hash is required" unless params.present? && params.is_a?(Hash)
+    end
+
+    def route
+      raise "Error: route method is required"
+    end
+  end
+end

data/lib/superset/chart/create.rb

@@ -0,0 +1,40 @@
+=begin
+Create a new chart from a set of params
+Suggestion is to base your params of an existing charts params and then modify them as needed
+So .. why not call the Superset::Chart::Duplicate class which then calls this Chart::Create class
+
+This class is a bit more generic and can be used to create a new chart from scratch (if your confident in the params)
+
+Usage: 
+Superset::Chart::Create.new(params: new_chart_params).perform
+=end
+
+module Superset
+  module Chart
+    class Create < Superset::Request
+
+      attr_reader :params
+
+      def initialize(params: )
+        @params = params
+      end
+
+      def perform
+        raise "Error: params hash is required" unless params.present? && params.is_a?(Hash)
+
+        logger.info("Creating New Chart")
+        response['id']
+      end
+
+      def response
+        @response ||= client.post(route, params)
+      end
+
+      private
+
+      def route
+        "chart/"
+      end
+    end
+  end
+end

data/lib/superset/chart/duplicate.rb

@@ -0,0 +1,75 @@
+# There is no API endpoint to duplicate charts in Superset. 
+# This class is a workaround.
+# Requires a source chart id, target dataset id
+
+module Superset
+  module Chart
+    class Duplicate < Superset::Request
+
+      attr_reader :source_chart_id, :target_dataset_id, :new_chart_name
+
+      def initialize(source_chart_id: , target_dataset_id: , new_chart_name:  )
+        @source_chart_id = source_chart_id
+        @target_dataset_id = target_dataset_id
+        @new_chart_name = new_chart_name
+      end
+
+      def perform
+        raise "Error: source_chart_id integer is required" unless source_chart_id.present? && source_chart_id.is_a?(Integer)
+        raise "Error: target_dataset_id integer is required" unless target_dataset_id.present? && target_dataset_id.is_a?(Integer)
+        raise "Error: new_chart_name string is required" unless new_chart_name.present? && new_chart_name.is_a?(String)
+
+        logger.info("Duplicating Chart #{source_chart_id}:#{source_chart['slice_name']}. New chart dataset #{target_dataset_id} and new chart name #{new_chart_name}")
+        Superset::Chart::Create.new(params: new_chart_params).perform
+      end
+
+      private
+
+    def new_chart_params
+      # pulled list from Swagger GUI for chart POST request
+      # commented out params seem to be not required .. figured out by trial and error
+      {
+        #"cache_timeout": 0,
+        #"certification_details": "string",
+        #"certified_by": "string",
+        #"dashboards": [ 0 ],
+        "datasource_id": target_dataset_id,
+    #   "datasource_name": new_chart_name,
+        "datasource_type": "table",
+    #    "description": "",
+    #    "external_url": "string",
+    #    "is_managed_externally": true,
+    #    "owners": [ 3 ],                          # TODO .. check if this is a Required attr, might need to get current API users id.
+        "params": new_chart_internal_params,
+        "query_context": new_chart_internal_query_context,
+        "query_context_generation": true,
+        "slice_name": new_chart_name,
+        "viz_type": source_chart['viz_type']
+      }
+    end
+
+    def new_chart_internal_params
+      new_params = JSON.parse(source_chart['params'])
+      new_params['datasource'] =  new_params['datasource'].gsub(source_chart_dataset_id.to_s, target_dataset_id.to_s)
+      new_params.delete('slice_id') # refers to the source chart id .. a new id will be generated in the new chart
+      new_params.to_json
+    end
+
+    def new_chart_internal_query_context
+      new_query_context = JSON.parse(source_chart['query_context'])
+      new_query_context['datasource'] =  new_query_context['datasource']['id'] = target_dataset_id
+      new_query_context['form_data']['datasource'] = new_query_context['form_data']['datasource'].gsub(source_chart_dataset_id.to_s, target_dataset_id.to_s)
+      new_query_context['form_data'].delete('slice_id')
+      new_query_context.to_json
+    end
+
+      def source_chart
+        @source_chart ||= Superset::Chart::Get.new(source_chart_id).result[0]
+      end
+
+      def source_chart_dataset_id
+        @source_chart_dataset_id ||= JSON.parse(source_chart[:query_context])['datasource']['id']
+      end
+    end
+  end
+end

data/lib/superset/chart/put.rb

@@ -0,0 +1,18 @@
+# Updates a chart in Superset with the given params
+#
+# Usage:
+# params = { owners: [ 58, 3 ] }
+# Superset::Chart::Put.new(object_id: 202, params: params ).perform
+
+module Superset
+  module Chart
+    class Put < Superset::BasePutRequest
+
+      private
+
+      def route
+        "chart/#{object_id}"
+      end
+    end
+  end
+end

data/lib/superset/chart/update_dataset.rb

@@ -43,7 +43,7 @@ module Superset
             new_params.merge!("query_context": query_context.to_json) # update to point to the new query context
             new_params.merge!("query_context_generation": true)            # new param set to true to regenerate the query context
           end
-         
+
           new_params
         end
       end

data/lib/superset/client.rb

@@ -44,7 +44,13 @@ module Superset
       @connection ||= Faraday.new(superset_host) do |f|
         f.authorization :Bearer, access_token
         f.use FaradayMiddleware::ParseJson, content_type: 'application/json'
-        f.request :json
+
+        if self.config.use_json
+          f.request :json
+        else
+          f.request :multipart
+          f.request :url_encoded
+        end
 
         f.adapter :net_http
       end

data/lib/superset/dashboard/bulk_delete_cascade.rb

@@ -31,7 +31,7 @@ module Superset
       private
 
       def delete_datasets(dashboard_id)
-        datasets_to_delete = Superset::Dashboard::Datasets::List.new(dashboard_id).datasets_details.map{|d| d[:id] }
+        datasets_to_delete = Superset::Dashboard::Datasets::List.new(dashboard_id: dashboard_id).datasets_details.map{|d| d[:id] }
         Superset::Dataset::BulkDelete.new(dataset_ids: datasets_to_delete).perform if datasets_to_delete.any?
       end
 

data/lib/superset/dashboard/compare.rb

@@ -37,8 +37,8 @@ module Superset
 
       def list_datasets
         puts "\n ====== DASHBOARD DATASETS ====== "
-        Superset::Dashboard::Datasets::List.new(first_dashboard_id).list
-        Superset::Dashboard::Datasets::List.new(second_dashboard_id).list
+        Superset::Dashboard::Datasets::List.new(dashboard_id: first_dashboard_id).list
+        Superset::Dashboard::Datasets::List.new(dashboard_id: second_dashboard_id).list
       end
 
       def list_charts

data/lib/superset/dashboard/datasets/list.rb

@@ -7,14 +7,15 @@ module Superset
   module Dashboard
     module Datasets
       class List < Superset::Request
-        attr_reader :id  # dashboard id
+        attr_reader :id, :include_filter_datasets # id - dashboard id
 
         def self.call(id)
           self.new(id).list
         end
 
-        def initialize(id)
-          @id = id
+        def initialize(dashboard_id:, include_filter_datasets: false)
+          @id = dashboard_id
+          @include_filter_datasets = include_filter_datasets
         end
 
         def perform
@@ -24,7 +25,7 @@ module Superset
 
         def schemas
           @schemas ||= begin
-            all_dashboard_schemas = result.map {|d| d[:schema] }.uniq
+            all_dashboard_schemas = datasets_details.map {|d| d[:schema] }.uniq
 
             # For the current superset setup we will assume a dashboard datasets will point to EXACTLY one schema, their own.
             # if not .. we need to know about it. Potentially we could override this check if others do not consider it a problem.
@@ -36,37 +37,64 @@ module Superset
         end
 
         def datasets_details
-          result.map do |details|
+          chart_datasets = result.map do |details|
             details.slice('id', 'datasource_name', 'schema', 'sql').merge('database' => details['database'].slice('id', 'name', 'backend')).with_indifferent_access
           end
+          return chart_datasets unless include_filter_datasets
+          chart_dataset_ids = chart_datasets.map{|d| d['id'] }
+          filter_dataset_ids_not_used_in_charts = filter_dataset_ids - chart_dataset_ids
+          return chart_datasets if filter_dataset_ids_not_used_in_charts.empty?
+          # returning chart and filter datasets
+          chart_datasets + filter_datasets(filter_dataset_ids_not_used_in_charts)
         end
 
         private
 
+        def filter_dataset_ids
+          @filter_dataset_ids ||= dashboard.filter_configuration.map { |c| c['targets'] }.flatten.compact.map { |c| c['datasetId'] }.flatten.compact.uniq
+        end
+
+        def filter_datasets(filter_dataset_ids_not_used_in_charts)
+          filter_dataset_ids_not_used_in_charts.map do |filter_dataset_id|
+            filter_dataset = Superset::Dataset::Get.new(filter_dataset_id).result
+            database_info = {
+              'id' => filter_dataset['database']['id'],
+              'name' => filter_dataset['database']['database_name'],
+              'backend' => filter_dataset['database']['backend']
+            }
+            filter_dataset.slice('id', 'datasource_name', 'schema', 'sql').merge('database' => database_info, 'filter_only': true).with_indifferent_access
+          end
+        end
+
         def route
           "dashboard/#{id}/datasets"
         end
 
         def list_attributes
-          ['id', 'datasource_name', 'database_id', 'database_name', 'database_backend', 'schema'].map(&:to_sym)
+          ['id', 'datasource_name', 'database_id', 'database_name', 'database_backend', 'schema', 'filter_only'].map(&:to_sym)
         end
 
         def rows
-          result.map do |d|
+          datasets_details.map do |d|
             [
               d[:id],
               d[:datasource_name],
               d[:database][:id],
               d[:database][:name],
               d[:database][:backend],
-              d[:schema]
+              d[:schema],
+              d[:filter_only]
             ]
           end
         end
 
         # when displaying a list of datasets, show dashboard title as well
         def title
-          @title ||= [id, Superset::Dashboard::Get.new(id).title].join(' ')
+          @title ||= [id, dashboard.title].join(' ')
+        end
+
+        def dashboard
+          @dashboard ||= Superset::Dashboard::Get.new(id)
         end
       end
     end

data/lib/superset/dashboard/embedded/get.rb

@@ -8,8 +8,8 @@ module Superset
           self.new(id).list
         end
 
-        def initialize(id)
-          @id = id
+        def initialize(dashboard_id:)
+          @id = dashboard_id
         end
 
         def response

data/lib/superset/dashboard/export.rb

@@ -13,7 +13,7 @@ module Superset
     class Export < Request
       include FileUtilities
 
-      TMP_SUPERSET_DASHBOARD_PATH = '/tmp/superset_dashboards'
+      TMP_SUPERSET_DASHBOARD_PATH = '/tmp/superset_dashboard_exports'
 
       attr_reader :dashboard_id, :destination_path
 
@@ -23,10 +23,18 @@ module Superset
       end
 
       def perform
+        logger.info("Exporting dashboard: #{dashboard_id}")
         create_tmp_dir
         save_exported_zip_file
         unzip_files
+        clean_destination_directory
         copy_export_files_to_destination_path if destination_path
+
+        Dir.glob("#{destination_path_with_dash_id}/**/*").select { |f| File.file?(f) }
+      rescue StandardError => e
+        raise
+      ensure
+        cleanup_temp_dir
       end
 
       def response
@@ -37,6 +45,10 @@ module Superset
         )
       end
 
+      def zip_file_name
+        @zip_file_name ||= "#{tmp_uniq_dashboard_path}/dashboard_#{dashboard_id}_export_#{datestamp}.zip"
+      end
+
       private
 
       def params
@@ -44,10 +56,12 @@ module Superset
       end
 
       def save_exported_zip_file
+        logger.info("Saving zip file: #{zip_file_name}")
         File.open(zip_file_name, 'wb') { |fp| fp.write(response.body) }
       end
 
       def unzip_files
+        logger.info("Unzipping file: #{zip_file_name}")
         @extracted_files = unzip_file(zip_file_name, tmp_uniq_dashboard_path)
       end
 
@@ -55,25 +69,43 @@ module Superset
         File.dirname(extracted_files[0])
       end
 
+      def destination_path_with_dash_id
+        @destination_path_with_dash_id ||= File.join(destination_path, dashboard_id.to_s)
+      end
+
+      def clean_destination_directory
+        logger.info("Cleaning destination directory: #{destination_path_with_dash_id}")
+        if Dir.exist?(destination_path_with_dash_id)
+          FileUtils.rm_rf(Dir.glob("#{destination_path_with_dash_id}/*"))
+        else
+          FileUtils.mkdir_p(destination_path_with_dash_id)
+        end
+      end
+
       def copy_export_files_to_destination_path
+        logger.info("Copying files to destination: #{destination_path_with_dash_id}")
         path_with_dash_id = File.join(destination_path, dashboard_id.to_s)
         FileUtils.mkdir_p(path_with_dash_id) unless File.directory?(path_with_dash_id)
+        FileUtils.cp(zip_file_name, path_with_dash_id)
 
         Dir.glob("#{download_folder}/*").each do |item|
           FileUtils.cp_r(item, path_with_dash_id)
         end
       end
 
-      def zip_file_name
-        @zip_file_name ||= "#{tmp_uniq_dashboard_path}/dashboard_#{dashboard_id}_export_#{datestamp}.zip"
+      def cleanup_temp_dir
+        if Dir.exist?(tmp_uniq_dashboard_path)
+          FileUtils.rm_rf(tmp_uniq_dashboard_path)
+        end
       end
 
       def create_tmp_dir
+        logger.info("Creating tmp directory: #{tmp_uniq_dashboard_path}")
         FileUtils.mkdir_p(tmp_uniq_dashboard_path) unless File.directory?(tmp_uniq_dashboard_path)
       end
 
       # uniq random tmp folder name for each export
-      # this will allow us to do a wildcard glop on the folder to get the files 
+      # this will allow us to do a wildcard glop on the folder to get the files
       def tmp_uniq_dashboard_path
         @tmp_uniq_dashboard_path ||= File.join(TMP_SUPERSET_DASHBOARD_PATH, uuid)
       end
@@ -93,6 +125,25 @@ module Superset
       def datestamp
         @datestamp ||= Time.now.strftime('%Y%m%d')
       end
+
+      def unzip_file(zip_path, destination)
+        extracted_files = []
+        Zip::File.open(zip_path) do |zip_file|
+          zip_file.each do |entry|
+            entry_path = File.join(destination, entry.name)
+            FileUtils.mkdir_p(File.dirname(entry_path))
+            zip_file.extract(entry, entry_path) unless File.exist?(entry_path)
+            extracted_files << entry_path
+          end
+        end
+        extracted_files
+      rescue => e
+        raise
+      end
+    end
+
+    def logger
+      @logger ||= Superset::Logger.new
     end
   end
-end
+end

data/lib/superset/dashboard/get.rb

@@ -4,6 +4,7 @@ module Superset
 
       attr_reader :id
 
+      # note .. this endpoint also accepts a dashboards uuid as the identifier
       def initialize(id)
         @id = id
       end
@@ -24,6 +25,10 @@ module Superset
       def json_metadata
         JSON.parse(result['json_metadata'])
       end
+      
+      def filter_configuration
+        json_metadata['native_filter_configuration'] || []
+      end
 
       def positions
         JSON.parse(result['position_json'])

data/lib/superset/dashboard/import.rb

@@ -0,0 +1,84 @@
+# Import the provided Dashboard zip file
+# In the context of this API import process, assumption is that the database.yaml file details will match
+# an existing database in the Target Superset Environment.
+
+# Scenario 1: Export from Env1 -- Import to Env1 into the SAME Environment
+#             Will result in updating/over writing the dashboard with the contents of the zip file 
+
+# Scenario 2: Export from Env1 -- Import to Env2 into a DIFFERENT Environment
+#             Assumption is that the database.yaml will match a database configuration in the target env. 
+#             Initial import will result in creating a new dashboard with the contents of the zip file. 
+#             Subsequent imports will result in updating/over writing the previous imported dashboard with the contents of the zip file.
+
+# the overwrite flag will determine if the dashboard will be updated or created new
+# overwrite: false .. will result in an error if a dashboard with the same UUID already exists
+
+# Usage
+# Superset::Dashboard::Import.new(source_zip_file: '/tmp/dashboard.zip').perform
+#
+
+module Superset
+  module Dashboard
+    class Import < Request
+      attr_reader :source_zip_file, :overwrite
+
+      def initialize(source_zip_file: , overwrite: true)
+        @source_zip_file = source_zip_file
+        @overwrite = overwrite
+      end
+
+      def perform
+        validate_params
+        response
+      end
+
+      def response
+        @response ||= client(use_json: false).post(
+          route,
+          payload
+        )
+      end
+
+      private
+
+      def validate_params
+        raise ArgumentError, 'source_zip_file is required' if source_zip_file.nil?
+        raise ArgumentError, 'source_zip_file does not exist' unless File.exist?(source_zip_file)
+        raise ArgumentError, 'source_zip_file is not a zip file' unless File.extname(source_zip_file) == '.zip'
+        raise ArgumentError, 'overwrite must be a boolean' unless [true, false].include?(overwrite)
+        raise ArgumentError, "zip target database does not exist: #{zip_database_config_not_found_in_superset}" if zip_database_config_not_found_in_superset.present?
+      end
+
+      def payload
+        {
+          formData: Faraday::UploadIO.new(source_zip_file, 'application/zip'),
+          overwrite: overwrite.to_s
+        }
+      end
+
+      def route
+        "dashboard/import/"
+      end
+
+      def zip_database_config_not_found_in_superset
+        zip_databases_details.select {|s| !superset_database_uuids_found.include?(s[:uuid]) }
+      end
+
+      def superset_database_uuids_found
+        @superset_database_uuids_found ||= begin
+          zip_databases_details.map {|i| i[:uuid]}.map do |uuid|
+            uuid if Superset::Database::List.new(uuid_equals: uuid).result.present?
+          end.compact
+        end
+      end
+
+      def zip_databases_details
+        zip_dashboard_config[:databases].map{|d| {uuid: d[:content][:uuid], name: d[:content][:database_name]} }
+      end
+
+      def zip_dashboard_config
+        @zip_dashboard_config ||= Superset::Services::DashboardLoader.new(dashboard_export_zip: source_zip_file).perform
+      end
+    end
+  end
+end