gooddata 2.1.19 → 2.2.0

data/lib/gooddata/models/dataset_mapping.rb

@@ -0,0 +1,36 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  class DatasetMapping
+    DATASET_MAPPING_GET_URI = '/gdc/dataload/projects/%<project_id>s/modelMapping/datasets'
+    DATASET_MAPPING_UPDATE_URI = '/gdc/dataload/projects/%<project_id>s/modelMapping/datasets/bulk/upsert'
+
+    class << self
+      def [](opts = { :client => GoodData.connection, :project => GoodData.project })
+        client, project = GoodData.get_client_and_project(opts)
+        get_uri = DATASET_MAPPING_GET_URI % { project_id: project.pid }
+        res = client.get(get_uri)
+        res
+      end
+
+      alias_method :get, :[]
+    end
+
+    def initialize(data)
+      @data = data
+    end
+
+    def save(opts)
+      client, project = GoodData.get_client_and_project(opts)
+
+      post_uri = DATASET_MAPPING_UPDATE_URI % { project_id: project.pid }
+      res = client.post(post_uri, @data, opts)
+      res
+    end
+  end
+end

data/lib/gooddata/models/metadata/label.rb

@@ -40,22 +40,20 @@ module GoodData
       end
     end
 
-    # Gets valid elements using /validElements? API
+    # Gets valid elements of a label for a specific paging (:offset and :limit) or get validElements of a specific value (:filter).
+    # In the case filter a specific value, because the API /validElements only filter by partial match, we need to filter again at client side for exact match.
     # @return [Array] Results
     def get_valid_elements(*args)
-      results, params = valid_elements(*args)
-      # TMA-775 - the validElements API can possibly return more matches than requested (usually 1)
-      # so we do a preliminary first request to check and then increase the limit if needed
-      if results['validElements']['paging']['total'].to_i != params[:limit]
+      if args && !args.empty? && args.first[:filter]
+        params = args.first
         params[:limit] = 100_000
         results, = valid_elements params
-        if params[:filter]
-          results['validElements']['items'] = results['validElements']['items'].reject do |i|
-            i['element']['title'] != params[:filter]
-          end
+        results['validElements']['items'] = results['validElements']['items'].select do |i|
+          i['element']['title'] == params[:filter]
         end
+      else
+        results, = valid_elements(*args)
       end
-
       results
     end
 
@@ -74,24 +72,25 @@ module GoodData
     # @option options [Number] :limit limits the number of values to certain number. Default is 100
     # @return [Array]
     def values(options = {})
-      Enumerator.new do |y|
-        offset = options[:offset] || 0
-        page_limit = options[:limit] || 100
-        loop do
-          results = get_valid_elements(limit: page_limit, offset: offset)
-
-          elements = results['validElements']
-          elements['items'].map do |el|
-            v = el['element']
-            y << {
-              :value => v['title'],
-              :uri => v['uri']
-            }
-          end
-          break if elements['items'].count < page_limit
-          offset += page_limit
+      all_values = []
+      offset = options[:offset] || 0
+      page_limit = options[:limit] || 100
+      loop do
+        results = get_valid_elements(limit: page_limit, offset: offset)
+
+        elements = results['validElements']
+        elements['items'].map do |el|
+          v = el['element']
+          all_values << {
+            :value => v['title'],
+            :uri => v['uri']
+          }
         end
+        break if elements['items'].count < page_limit
+
+        offset += page_limit
       end
+      all_values
     end
 
     def values_count
@@ -136,7 +135,7 @@ module GoodData
       if status_url
         results = client.poll_on_response(status_url) do |body|
           status = body['taskState'] && body['taskState']['status']
-          status == 'RUNNING' || status == 'PREPARED'
+          status == 'RUNNING' || status == 'PREPARED' || body['uri']
         end
       end
 

data/lib/gooddata/models/project.rb

@@ -30,6 +30,7 @@ require_relative 'process'
 require_relative 'project_log_formatter'
 require_relative 'project_role'
 require_relative 'blueprint/blueprint'
+require_relative 'dataset_mapping'
 
 require_relative 'metadata/scheduled_mail'
 require_relative 'metadata/scheduled_mail/dashboard_attachment'
@@ -255,6 +256,22 @@ module GoodData
         transfer_schedules(from_project, to_project)
       end
 
+      def get_dataset_mapping(from_project)
+        GoodData::DatasetMapping.get(:client => from_project.client, :project => from_project)
+      end
+
+      def update_dataset_mapping(model_mapping_json, to_project)
+        dataset_mapping = GoodData::DatasetMapping.new(model_mapping_json)
+        res = dataset_mapping.save(:client => to_project.client, :project => to_project)
+        status = res&.dig('datasetMappings', 'items').nil? ? "Failed" : "OK"
+        count = "OK".eql?(status) ? res['datasetMappings']['items'].length : 0
+        {
+          to: to_project.pid,
+          count: count,
+          status: status
+        }
+      end
+
       # @param from_project The source project
       # @param to_project The target project
       # @param options Optional parameters
@@ -337,20 +354,16 @@ module GoodData
       def get_data_source_alias(data_source_id, client, aliases)
         unless aliases[data_source_id]
           data_source = GoodData::DataSource.from_id(data_source_id, client: client)
-          if data_source&.dig('dataSource', 'alias')
+          if data_source&.alias
             aliases[data_source_id] = {
-              :type => get_data_source_type(data_source),
-              :alias => data_source['dataSource']['alias']
+              :type => data_source.type,
+              :alias => data_source.alias
             }
           end
         end
         aliases[data_source_id]
       end
 
-      def get_data_source_type(data_source_data)
-        data_source_data&.dig('dataSource', 'connectionInfo') ? data_source_data['dataSource']['connectionInfo'].first[0].upcase : ""
-      end
-
       def replace_process_data_source_ids(process_data, client, aliases)
         component = process_data.dig(:process, :component)
         if component&.dig(:configLocation, :dataSourceConfig)
@@ -460,7 +473,9 @@ module GoodData
         local_stuff = local_schedules.map do |s|
           v = s.to_hash
           after_schedule = local_schedules.find { |s2| s.trigger_id == s2.obj_id }
-          v[:after] = s.trigger_id && after_schedule && after_schedule.name
+          after_process_schedule = from_project_processes.find { |p| after_schedule && p.obj_id == after_schedule.process_id }
+          v[:after] = s.trigger_id && after_process_schedule && after_schedule && after_schedule.name
+          v[:trigger_execution_status] = s.trigger_execution_status
           v[:remote_schedule] = s
           v[:params] = v[:params].except("EXECUTABLE", "PROCESS_ID")
           v.compact
@@ -529,6 +544,7 @@ module GoodData
             schedule.params = (schedule_spec[:params] || {})
             schedule.cron = schedule_spec[:cron] if schedule_spec[:cron]
             schedule.after = schedule_cache[schedule_spec[:after]] if schedule_spec[:after]
+            schedule.trigger_execution_status = schedule_cache[schedule_spec[:trigger_execution_status]] if schedule_spec[:after]
             schedule.hidden_params = schedule_spec[:hidden_params] || {}
             if process_spec.type != :dataload
               schedule.executable = schedule_spec[:executable] || (process_spec.type == :ruby ? 'main.rb' : 'main.grf')
@@ -589,7 +605,8 @@ module GoodData
           hidden_params: schedule_spec[:hidden_params],
           name: schedule_spec[:name],
           reschedule: schedule_spec[:reschedule],
-          state: schedule_spec[:state]
+          state: schedule_spec[:state],
+          trigger_execution_status: schedule_spec[:trigger_execution_status]
         }
       end
     end
@@ -2022,6 +2039,14 @@ module GoodData
       GoodData::Project.transfer_etl(client, self, target)
     end
 
+    def dataset_mapping
+      GoodData::Project.get_dataset_mapping(self)
+    end
+
+    def update_dataset_mapping(model_mapping_json)
+      GoodData::Project.update_dataset_mapping(model_mapping_json, self)
+    end
+
     def transfer_processes(target)
       GoodData::Project.transfer_processes(self, target)
     end

data/lib/gooddata/models/schedule.rb

@@ -101,6 +101,7 @@ module GoodData
 
         schedule.name = options[:name]
         schedule.set_trigger(trigger)
+        schedule.trigger_execution_status = options[:trigger_execution_status]
         schedule.params = default_opts[:params].merge(options[:params] || {})
         schedule.hidden_params = options[:hidden_params] || {}
         schedule.timezone = options[:timezone] || default_opts[:timezone]
@@ -468,6 +469,7 @@ module GoodData
         hidden_params: hidden_params,
         cron: cron,
         trigger_id: trigger_id,
+        trigger_execution_status: trigger_execution_status,
         timezone: timezone,
         uri: uri,
         reschedule: reschedule,
@@ -486,6 +488,16 @@ module GoodData
       self
     end
 
+    def trigger_execution_status
+      json['schedule']['triggerExecutionStatus']
+    end
+
+    def trigger_execution_status=(trigger_execution_status)
+      json['schedule']['triggerExecutionStatus'] = trigger_execution_status
+      @dirty = true
+      self # rubocop:disable Lint/Void
+    end
+
     def name
       json['schedule']['name']
     end
@@ -530,7 +542,7 @@ module GoodData
           'hiddenParams' => GoodData::Helpers.encode_hidden_params(hidden_params)
         }
       }
-
+      res['schedule']['triggerExecutionStatus'] = trigger_execution_status if trigger_execution_status
       res['schedule']['reschedule'] = reschedule if reschedule
 
       res

data/lib/gooddata/models/user_filters/user_filter_builder.rb

@@ -203,7 +203,7 @@ module GoodData
     # so it precaches the values and still be able to function for larger ones even
     # though that would mean tons of requests
     def self.get_small_labels(labels_cache)
-      labels_cache.values.select { |label| label && label.values_count && label.values_count < 100_000 }
+      labels_cache.values.select { |label| label &.values_count &. < 100_000 }
     end
 
     # Creates a MAQL expression(s) based on the filter defintion.
@@ -421,68 +421,73 @@ module GoodData
                  results: create_results + delete_results }
       end
 
-      create_results = to_create.each_slice(100).flat_map do |batch|
-        batch.pmapcat do |related_uri, group|
-          group.each(&:save)
-          res = client.get("/gdc/md/#{project.pid}/userfilters?users=#{related_uri}")
-          items = res['userFilters']['items'].empty? ? [] : res['userFilters']['items'].first['userFilters']
-
-          payload = {
-            'userFilters' => {
-              'items' => [{
-                'user' => related_uri,
-                'userFilters' => items.concat(group.map(&:uri))
-              }]
+      if to_create.empty?
+        create_results = []
+      else
+        create_results = to_create.each_slice(100).flat_map do |batch|
+          batch.pmapcat do |related_uri, group|
+            group.each(&:save)
+            res = client.get("/gdc/md/#{project.pid}/userfilters?users=#{related_uri}")
+            items = res['userFilters']['items'].empty? ? [] : res['userFilters']['items'].first['userFilters']
+
+            payload = {
+              'userFilters' => {
+                'items' => [{
+                  'user' => related_uri,
+                  'userFilters' => items.concat(group.map(&:uri))
+                }]
+              }
             }
-          }
-          res = client.post("/gdc/md/#{project.pid}/userfilters", payload)
+            res = client.post("/gdc/md/#{project.pid}/userfilters", payload)
 
-          # turn the errors from hashes into array of hashes
-          update_result = res['userFiltersUpdateResult'].flat_map do |k, v|
-            v.map { |r| { status: k.to_sym, user: r, type: :create } }
-          end
+            # turn the errors from hashes into array of hashes
+            update_result = res['userFiltersUpdateResult'].flat_map do |k, v|
+              v.map { |r| { status: k.to_sym, user: r, type: :create } }
+            end
 
-          update_result.map do |result|
-            result[:status] == :failed ? result.merge(GoodData::Helpers.symbolize_keys(result[:user])) : result
+            update_result.map do |result|
+              result[:status] == :failed ? result.merge(GoodData::Helpers.symbolize_keys(result[:user])) : result
+            end
           end
         end
+        project_log_formatter.log_user_filter_results(create_results, to_create)
+        create_errors = create_results.select { |r| r[:status] == :failed }
+        fail "Creating MUFs resulted in errors: #{create_errors}" if create_errors.any?
       end
 
-      project_log_formatter.log_user_filter_results(create_results, to_create)
-      create_errors = create_results.select { |r| r[:status] == :failed }
-      fail "Creating MUFs resulted in errors: #{create_errors}" if create_errors.any?
-
-      delete_results = unless options[:do_not_touch_filters_that_are_not_mentioned]
-                         to_delete.each_slice(100).flat_map do |batch|
-                           batch.flat_map do |related_uri, group|
-                             results = []
-                             if related_uri
-                               res = client.get("/gdc/md/#{project.pid}/userfilters?users=#{related_uri}")
-                               items = res['userFilters']['items'].empty? ? [] : res['userFilters']['items'].first['userFilters']
-                               payload = {
-                                 'userFilters' => {
-                                   'items' => [
-                                     {
-                                       'user' => related_uri,
-                                       'userFilters' => items - group.map(&:uri)
-                                     }
-                                   ]
-                                 }
-                               }
-                               res = client.post("/gdc/md/#{project.pid}/userfilters", payload)
-                               results.concat(res['userFiltersUpdateResult']
+      if to_delete.empty?
+        delete_results = []
+      elsif !options[:do_not_touch_filters_that_are_not_mentioned]
+        delete_results = to_delete.each_slice(100).flat_map do |batch|
+          batch.flat_map do |related_uri, group|
+            results = []
+            if related_uri
+              res = client.get("/gdc/md/#{project.pid}/userfilters?users=#{related_uri}")
+              items = res['userFilters']['items'].empty? ? [] : res['userFilters']['items'].first['userFilters']
+              payload = {
+                'userFilters' => {
+                  'items' => [
+                    {
+                      'user' => related_uri,
+                      'userFilters' => items - group.map(&:uri)
+                    }
+                  ]
+                }
+              }
+              res = client.post("/gdc/md/#{project.pid}/userfilters", payload)
+              results.concat(res['userFiltersUpdateResult']
                                  .flat_map { |k, v| v.map { |r| { status: k.to_sym, user: r, type: :delete } } }
                                  .map { |result| result[:status] == :failed ? result.merge(GoodData::Helpers.symbolize_keys(result[:user])) : result })
-                             end
-                             group.peach(&:delete)
-                             results
-                           end
-                         end
-                       end
+            end
+            group.peach(&:delete)
+            results
+          end
 
-      project_log_formatter.log_user_filter_results(delete_results, to_delete)
-      delete_errors = delete_results.select { |r| r[:status] == :failed } if delete_results
-      fail "Deleting MUFs resulted in errors: #{delete_errors}" if delete_errors && delete_errors.any?
+          project_log_formatter.log_user_filter_results(delete_results, to_delete)
+          delete_errors = delete_results.select { |r| r[:status] == :failed } if delete_results
+          fail "Deleting MUFs resulted in errors: #{delete_errors}" if delete_errors&.any?
+        end
+      end
 
       { created: to_create, deleted: to_delete, results: create_results + (delete_results || []) }
     end

data/lib/gooddata/rest/phmap.rb

@@ -82,6 +82,7 @@ module GoodData
 
         ['/gdc/internal/projects/{id}/objects/setPermissions', %r{/gdc/internal/projects/[^\/]+/objects/setPermissions}],
         ['/gdc/internal/projects/{id}/roles', %r{/gdc/internal/projects/[^\/]+/roles}],
+        ['/gdc/dataload/projects/{id}/modelMapping/datasets/bulk/upsert', %r{/gdc/dataload/projects/[^\/]+/modelMapping/datasets/bulk/upsert}],
         ['/gdc/md/{id}/variables/item/{id}', %r{/gdc/md/[^\/]+/variables/item/[\d]+}],
         ['/gdc/md/{id}/validate/task/{id}', %r{/gdc/md/[^\/]+/validate/task/[^\/]+}],
         ['/gdc/md/{id}/using2/{id}/{id}', %r{/gdc/md/[^\/]+/using2/[\d]+/[\d]+}],

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gooddata
 version: !ruby/object:Gem::Version
-  version: 2.1.19
+  version: 2.2.0
 platform: ruby
 authors:
 - Pavel Kolesnikov
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-21 00:00:00.000000000 Z
+date: 2022-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: license_finder
@@ -336,6 +336,34 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.4.0
+- !ruby/object:Gem::Dependency
+  name: azure-storage-blob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.10.0
 - !ruby/object:Gem::Dependency
   name: gli
   requirement: !ruby/object:Gem::Requirement
@@ -488,20 +516,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.1
-- !ruby/object:Gem::Dependency
-  name: salesforce_bulk_query
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
@@ -584,6 +598,8 @@ files:
 - ".flayignore"
 - ".gdc-ii-config-chart.yaml"
 - ".gdc-ii-config.yaml"
+- ".github/workflows/build.yml"
+- ".github/workflows/pre-merge.yml"
 - ".gitignore"
 - ".pronto.yml"
 - ".rspec"
@@ -623,6 +639,8 @@ files:
 - bin/users.sh
 - ci.rake
 - ci/bigquery/pom.xml
+- ci/mssql/pom.xml
+- ci/mysql/pom.xml
 - ci/postgresql/pom.xml
 - ci/redshift/pom.xml
 - ci/snowflake/pom.xml
@@ -651,7 +669,6 @@ files:
 - lib/gooddata/bricks/middleware/aws_middleware.rb
 - lib/gooddata/bricks/middleware/base_middleware.rb
 - lib/gooddata/bricks/middleware/bench_middleware.rb
-- lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb
 - lib/gooddata/bricks/middleware/context_logger_decorator.rb
 - lib/gooddata/bricks/middleware/context_manager.rb
 - lib/gooddata/bricks/middleware/decode_params_middleware.rb
@@ -686,9 +703,14 @@ files:
 - lib/gooddata/client.rb
 - lib/gooddata/cloud_resources/bigquery/bigquery_client.rb
 - lib/gooddata/cloud_resources/bigquery/drivers/.gitkeepme
+- lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb
 - lib/gooddata/cloud_resources/cloud_resource_client.rb
 - lib/gooddata/cloud_resources/cloud_resource_factory.rb
 - lib/gooddata/cloud_resources/cloud_resources.rb
+- lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme
+- lib/gooddata/cloud_resources/mssql/mssql_client.rb
+- lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme
+- lib/gooddata/cloud_resources/mysql/mysql_client.rb
 - lib/gooddata/cloud_resources/postgresql/drivers/.gitkeepme
 - lib/gooddata/cloud_resources/postgresql/postgresql_client.rb
 - lib/gooddata/cloud_resources/redshift/drivers/log4j.properties
@@ -790,6 +812,7 @@ files:
 - lib/gooddata/lcm/actions/synchronize_cas.rb
 - lib/gooddata/lcm/actions/synchronize_clients.rb
 - lib/gooddata/lcm/actions/synchronize_color_palette.rb
+- lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb
 - lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb
 - lib/gooddata/lcm/actions/synchronize_label_types.rb
 - lib/gooddata/lcm/actions/synchronize_ldm.rb
@@ -801,9 +824,11 @@ files:
 - lib/gooddata/lcm/actions/synchronize_user_filters.rb
 - lib/gooddata/lcm/actions/synchronize_user_groups.rb
 - lib/gooddata/lcm/actions/synchronize_users.rb
+- lib/gooddata/lcm/actions/update_metric_formats.rb
 - lib/gooddata/lcm/actions/update_release_table.rb
 - lib/gooddata/lcm/brick_logger.rb
 - lib/gooddata/lcm/data/create_lcm_release.sql.erb
+- lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb
 - lib/gooddata/lcm/data/insert_into_lcm_release.sql.erb
 - lib/gooddata/lcm/data/select_from_lcm_release.sql.erb
 - lib/gooddata/lcm/data/update_lcm_release.sql.erb
@@ -917,6 +942,7 @@ files:
 - lib/gooddata/models/client_synchronization_result_details.rb
 - lib/gooddata/models/data_product.rb
 - lib/gooddata/models/data_source.rb
+- lib/gooddata/models/dataset_mapping.rb
 - lib/gooddata/models/datawarehouse.rb
 - lib/gooddata/models/domain.rb
 - lib/gooddata/models/execution.rb
@@ -1016,7 +1042,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubyforge_project: 
+rubygems_version: 2.5.2.3
 signing_key: 
 specification_version: 4
 summary: A convenient Ruby wrapper around the GoodData RESTful API

data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb

@@ -1,37 +0,0 @@
-# encoding: UTF-8
-#
-# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree.
-
-require 'salesforce_bulk_query'
-
-require_relative 'base_middleware'
-
-module GoodData
-  module Bricks
-    class BulkSalesforceMiddleware < Bricks::Middleware
-      DEFAULT_VERSION = '29.0'.freeze
-
-      def self.create_client(params)
-        salesforce = nil
-        if params['salesforce_client']
-
-          client = params['salesforce_client']
-          client.authenticate!
-
-          salesforce = SalesforceBulkQuery::Api.new(client, logger: params['GDC_LOGGER'])
-          # SalesforceBulkQuery adds its own Restforce logging so turn it off
-          Restforce.log = false if params['GDC_LOGGER']
-        end
-        params.merge('salesforce_bulk_client' => salesforce)
-      end
-
-      def call(params)
-        params = params.to_hash
-        params = self.class.create_client(params)
-        @app.call(params)
-      end
-    end
-  end
-end