gooddata 2.1.12 → 2.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21bb1e8b1a4eacb96fbf7aa25e22b54927a881fe3e04648f886b3ba132110c18
-  data.tar.gz: 797c83b8e0c76d07b4ca976023ff58cc82a1fa1bc3e63ad68fee294e24f9ddde
+  metadata.gz: 8ab34a80c9d1b2d1d7608b70b767f3801fc1f47538b5d87678c83820f86765b1
+  data.tar.gz: f44173431a6599fcc7854e81027ca516ede3b24fba6f9720035295c52e05f821
 SHA512:
-  metadata.gz: bb2d421d223db058c231ec0d66b3325a97237a5dc050305e17bc27dcab78694c8237241fbca2dab60d45a8257cd5b8bfb875c2db532e38fa075cfc02cfb57b26
-  data.tar.gz: 7ef77abd3d61fe54238b25b00a8d4ea05852fbc3474ae64aa54a770b739f3d64a5cc847e42cc993ac232423204ac9a354a4401882663dd8723a02f95fb1e91ba
+  metadata.gz: ed09925ea424e7ffb355eca2c9a1a769231d170b0c266cd858c0ab22076f90cf77e5123de9f1f45d768e17445397fa054fce480f52fb1dcec6193bd35198501f
+  data.tar.gz: 6f31fb629a09bddacd4a256c4ec660f589918e7aa712a375b58457c42789756cfc0b462782b8bf479dc6e98e8e9e91d64ce4e816a7396cf785dfcf5a47ac1fcc

data/CHANGELOG.md

@@ -1,4 +1,11 @@
 # GoodData Ruby SDK Changelog
+## 2.1.13
+ - FEATURE: TMA-1676 Support LCM release across domain
+ - FEATURE: TMA-1672 Support sync process with generic datasource
+ - FEATURE: MSF-17743 upgrade custom v2 for rollout brick
+ - BUGFIX: MSF-17975 Introduce gdcshare to lcm bricks
+ - BUGFIX: TMA-1673 Update params processing to accept dot and space
+
 ## 2.1.12
  - FEATURE: MSF-17621 Apply patched version for activesupport to fix vulnerable issue
  - CONFIG: SETI-4379 Add gdc-fossa configuration for gooddata-ruby

data/Dockerfile

@@ -37,7 +37,9 @@ RUN rvm install jruby-${JRUBY_VERSION} && gem update --system \
 WORKDIR /src
 
 RUN groupadd -g 48 apache \
+    && groupadd -g 65065 gdcshare \
     && useradd -u 48 -m --no-log-init -r -g apache -G rvm apache \
+    && usermod -a -G gdcshare apache \
     && chown apache: /src
 
 USER apache

data/README.md

@@ -9,6 +9,8 @@ The best documentation for the GoodData API can be found using these resources:
  * http://developer.gooddata.com/api
  * https://secure.gooddata.com/gdc
  * http://rubydoc.info/gems/gooddata/frames
+ 
+Feel free to check out the [GoodData community website](http://community.gooddata.com/) if you have any questions about the GoodData Analytics platform, our API, or this library.
 
 ## Status
 

data/SDK_VERSION

@@ -1 +1 @@
-2.1.12
+2.1.13

data/VERSION

@@ -1 +1 @@
-3.7.20
+3.7.22

data/lib/gooddata/helpers/data_source_helpers.rb

@@ -0,0 +1,47 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2020 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+# frozen_string_literal: false
+
+module GoodData
+  module Helpers
+    class << self
+      # Get a data source information from server by id
+      #
+      # @param [String] data_source_id The data source ID
+      # @param [Object] client The Rest Client object
+      # @return [Hash] Returns Data source
+      def get_data_source_by_id(data_source_id, client)
+        unless data_source_id.blank?
+          uri = "/gdc/dataload/dataSources/#{data_source_id}"
+          client.get(uri)
+        end
+      end
+
+      # Verify to see if the data source exists in the domain using its alias
+      #
+      # @param [String] ds_alias The data source's alias
+      # @param [Object] client The Rest Client object
+      # @return [String] Id of the data source or failed with the reason
+      def verify_data_source_alias(ds_alias, client)
+        domain = client.connection.server.url
+        fail "The data source alias is empty, check your data source configuration." unless ds_alias
+
+        uri = "/gdc/dataload/dataSources/internal/availableAlias?alias=#{ds_alias[:alias]}"
+        res = client.get(uri)
+        fail "Unable to get information about the Data Source '#{ds_alias[:alias]}' in the domain '#{domain}'" unless res
+        fail "Unable to find the #{ds_alias[:type]} Data Source '#{ds_alias[:alias]}' in the domain '#{domain}'" if res['availableAlias']['available']
+
+        ds_type = res['availableAlias']['existingDataSource']['type']
+        if ds_type && ds_type != ds_alias[:type]
+          fail "Wrong Data Source type - the '#{ds_type}' type is expected but the Data Source '#{ds_alias[:alias]}' in the domain '#{domain}' has the '#{ds_alias[:type]}' type"
+        else
+          res['availableAlias']['existingDataSource']['id']
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/helpers/global_helpers_params.rb

@@ -242,7 +242,7 @@ module GoodData
 
       def resolve_reference_params(data_params, params)
         reference_values = []
-        regexps = Regexp.union(/\\\\/, /\\\$/, /\$\{(\w+)\}/)
+        regexps = Regexp.union(/\\\\/, /\\\$/, /\$\{([\w\s\.]+)\}/)
         resolve_reference = lambda do |v|
           if v.is_a? Hash
             Hash[
@@ -262,7 +262,7 @@ module GoodData
                 data_params.is_a?(Hash) ? '\\' : '\\\\' # rubocop: disable Metrics/BlockNesting
               elsif match =~ /\\\$/
                 '$'
-              elsif match =~ /\$\{(\w+)\}/
+              elsif match =~ /\$\{([\w\s\.]+)\}/
                 val = params["#{$1}"]
                 if val
                   reference_values << val

data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+# (C) 2019-2020 GoodData Corporation
+require_relative 'base_action'
+
+# Migrate date dimension urn:gooddata:date or urn:custom:date to urn:custom_v2:date
+module GoodData
+  module LCM2
+    class MigrateGdcDateDimension < BaseAction
+      DESCRIPTION = 'Migrate Gdc Date Dimension'
+      DATE_DIMENSION_CUSTOM_V2 = 'urn:custom_v2:date'
+      DATE_DIMENSION_OLD = %w[urn:gooddata:date urn:custom:date]
+
+      PARAMS = define_params(self) do
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+
+        description 'Specifies how to synchronize LDM and resolve possible conflicts'
+        param :synchronize_ldm, instance_of(Type::SynchronizeLDM), required: false, default: 'diff_against_master_with_fallback'
+
+        description 'Synchronization Info'
+        param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
+      end
+
+      RESULT_HEADER = %i[from to status]
+
+      class << self
+        def call(params)
+          results = []
+          params.synchronize.map do |segment_info|
+            result = migrate_date_dimension(params, segment_info)
+            results.concat(result)
+          end
+
+          {
+            results: results
+          }
+        end
+
+        def migrate_date_dimension(params, segment_info)
+          results = []
+          client = params.gdc_gd_client
+          latest_blueprint = segment_info[:from_blueprint]
+          # don't migrate when latest master doesn't contain custom v2 date.
+          return results unless contain_v2?(latest_blueprint)
+
+          previous_blueprint = segment_info[:previous_master]&.blueprint
+          # check latest master and previous master
+          master_upgrade_datasets = get_upgrade_dates(latest_blueprint, previous_blueprint) if params[:synchronize_ldm].downcase == 'diff_against_master' && previous_blueprint
+          unless master_upgrade_datasets&.empty?
+            segment_info[:to].pmap do |entry|
+              pid = entry[:pid]
+              to_project = client.projects(pid) || fail("Invalid 'to' project specified - '#{pid}'")
+              to_blueprint = to_project.blueprint
+              upgrade_datasets = get_upgrade_dates(latest_blueprint, to_blueprint)
+              next if upgrade_datasets.empty?
+
+              message = get_upgrade_message(upgrade_datasets)
+
+              results << {
+                from: segment_info[:from],
+                to: pid,
+                status: to_project.upgrade_custom_v2(message)
+              }
+            end
+          end
+
+          results
+        end
+
+        def get_upgrade_dates(src_blueprint, dest_blueprint)
+          dest_dates = get_date_dimensions(dest_blueprint) if dest_blueprint
+          src_dates = get_date_dimensions(src_blueprint) if src_blueprint
+
+          return false if dest_dates.empty? || src_dates.empty?
+
+          upgrade_datasets = []
+          dest_dates.each do |dest|
+            src_dim = get_date_dimension(src_blueprint, dest[:id])
+            next unless src_dim
+
+            upgrade_datasets << src_dim[:identifier] if upgrade?(src_dim, dest) && src_dim[:identifier]
+          end
+
+          upgrade_datasets
+        end
+
+        def get_upgrade_message(upgrade_datasets)
+          {
+            upgrade: {
+              dateDatasets: {
+                upgrade: "exact",
+                datasets: upgrade_datasets
+              }
+            }
+          }
+        end
+
+        def upgrade?(src_dim, dest_dim)
+          src_dim[:urn] == DATE_DIMENSION_CUSTOM_V2 && DATE_DIMENSION_OLD.any? { |e| dest_dim[:urn] == e }
+        end
+
+        def contain_v2?(blueprint)
+          get_date_dimensions(blueprint).any? { |e| e[:urn] == DATE_DIMENSION_CUSTOM_V2 }
+        end
+
+        def get_date_dimension(blueprint, id)
+          GoodData::Model::ProjectBlueprint.find_date_dimension(blueprint, id)
+        end
+
+        def get_date_dimensions(blueprint)
+          GoodData::Model::ProjectBlueprint.date_dimensions(blueprint)
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/synchronize_ldm.rb

@@ -49,6 +49,8 @@ module GoodData
         param :include_deprecated, instance_of(Type::BooleanType), required: false, default: false
       end
 
+      RESULT_HEADER = %i[from to status]
+
       class << self
         def call(params)
           results = []
@@ -76,9 +78,9 @@ module GoodData
           include_deprecated = params.include_deprecated.to_b
           from_pid = segment_info[:from]
           from = params.development_client.projects(from_pid) || fail("Invalid 'from' project specified - '#{from_pid}'")
-
           GoodData.logger.info "Creating Blueprint, project: '#{from.title}', PID: #{from_pid}"
           blueprint = from.blueprint(include_ca: params.include_computed_attributes.to_b)
+          segment_info[:from_blueprint] = blueprint
           maql_diff = nil
           previous_master = segment_info[:previous_master]
           diff_against_master = %w(diff_against_master_with_fallback diff_against_master)

data/lib/gooddata/lcm/lcm2.rb

@@ -138,6 +138,7 @@ module GoodData
         EnsureTechnicalUsersDomain,
         EnsureTechnicalUsersProject,
         SynchronizeLdm,
+        MigrateGdcDateDimension,
         SynchronizeClients,
         SynchronizeComputedAttributes,
         CollectDymanicScheduleParams,

data/lib/gooddata/models/from_wire.rb

@@ -105,6 +105,7 @@ module GoodData
           d[:title] = date_dim['dateDimension']['title']
           d[:urn] = date_dim['dateDimension']['urn']
           d[:identifier_prefix] = date_dim['dateDimension']['identifierPrefix']
+          d[:identifier] = date_dim['dateDimension']['identifier'] if date_dim['dateDimension']['identifier']
           d[:columns] = parse_bridges(date_dim)
         end
       end

data/lib/gooddata/models/process.rb

@@ -118,11 +118,13 @@ module GoodData
         GoodData.logger.info("Deploying #{path}") if verbose
 
         deployed_path = Process.upload_package(path, files_to_exclude, client: client, project: project)
+        data_sources = options[:data_sources] || []
         data = {
           :process => {
             :name => deploy_name,
             :path => "/uploads/#{File.basename(deployed_path)}",
-            :type => type
+            :type => type,
+            :dataSources => data_sources
           }
         }
 
@@ -171,10 +173,12 @@ module GoodData
         verbose = options[:verbose] || false
         GoodData.logger.info("Deploying #{path}") if verbose
 
+        data_sources = options[:data_sources] || []
         data = {
           process: {
             name: deploy_name,
             path: path,
+            dataSources: data_sources,
             type: 'RUBY'
           }
         }
@@ -185,7 +189,7 @@ module GoodData
       def deploy_component(data, options = { client: GoodData.client, project: GoodData.project })
         client, project = GoodData.get_client_and_project(options)
         data = { process: data } unless data[:process]
-        data[:process] = GoodData::Helpers.symbolize_keys(data[:process]).select { |k| %i[type name component].include? k }
+        data[:process] = GoodData::Helpers.symbolize_keys(data[:process]).select { |k| %i[type name component dataSources].include? k }
         data[:process][:component] = GoodData::Helpers.symbolize_keys(data[:process][:component]).select { |k| %i[name version configLocation config].include? k }
 
         save(data, options)
@@ -266,7 +270,7 @@ module GoodData
     # @option options [String] :name Readable name of the process
     # @option options [Boolean] :verbose (false) Switch on verbose mode for detailed logging
     def deploy(path, options = {})
-      Process.deploy(path, { client: client, process_id: process_id, :project => project, :name => name, :type => type }.merge(options))
+      Process.deploy(path, { client: client, process_id: process_id, :project => project, :name => name, :type => type, :data_sources => data_sources }.merge(options))
     end
 
     # Downloads the process from S3 in a zipped form.
@@ -326,6 +330,10 @@ module GoodData
       process['component']
     end
 
+    def data_sources
+      process['dataSources']
+    end
+
     # Determines whether the process is an ADDv2 component.
     # @return [Bool] True if the process is an ADDv2 component.
     def add_v2_component?

data/lib/gooddata/models/project.rb

@@ -261,21 +261,26 @@ module GoodData
       # @option ads_output_stage_uri Uri of the source output stage. It must be in the same domain as the target project.
       def transfer_processes(from_project, to_project, options = {})
         options = GoodData::Helpers.symbolize_keys(options)
+        aliases = {}
         to_project_processes = to_project.processes
         additional_hidden_params = options[:additional_hidden_params] || {}
         result = from_project.processes.uniq(&:name).map do |process|
-          fail "The process name #{process.name} must be unique in transfered project #{to_project}" if to_project_processes.count { |p| p.name == process.name } > 1
+          fail "The process name #{process.name} must be unique in transferred project #{to_project}" if to_project_processes.count { |p| p.name == process.name } > 1
           next if process.type == :dataload || process.add_v2_component?
+          collect_process_aliases(process.data, from_project.client, aliases)
 
           to_process = to_project_processes.find { |p| p.name == process.name }
 
+          data_sources = GoodData::Helpers.symbolize_keys_recursively!(process.data_sources)
+          data_sources = replace_data_source_ids(data_sources, to_project.client, aliases)
           to_process = if process.path
                          to_process.delete if to_process
-                         GoodData::Process.deploy_from_appstore(process.path, name: process.name, client: to_project.client, project: to_project)
+                         Process.deploy_from_appstore(process.path, name: process.name, client: to_project.client, project: to_project, data_sources: data_sources)
                        elsif process.component
                          to_process.delete if to_process
                          process_hash = GoodData::Helpers::DeepMergeableHash[GoodData::Helpers.symbolize_keys(process.to_hash)].deep_merge(additional_hidden_params)
-                         GoodData::Process.deploy_component(process_hash, project: to_project, client: to_project.client)
+                         process_hash = replace_process_data_source_ids(process_hash, to_project.client, aliases)
+                         Process.deploy_component(process_hash, project: to_project, client: to_project.client)
                        else
                          Dir.mktmpdir('etl_transfer') do |dir|
                            dir = Pathname(dir)
@@ -283,11 +288,10 @@ module GoodData
                            File.open(filename, 'w') do |f|
                              f << process.download
                            end
-
                            if to_process
-                             to_process.deploy(filename, type: process.type, name: process.name)
+                             to_process.deploy(filename, type: process.type, name: process.name, data_sources: data_sources)
                            else
-                             to_project.deploy_process(filename, type: process.type, name: process.name)
+                             to_project.deploy_process(filename, type: process.type, name: process.name, data_sources: data_sources)
                            end
                          end
                        end
@@ -318,6 +322,56 @@ module GoodData
         result.compact
       end
 
+      def collect_process_aliases(process_data, client, aliases)
+        data_sources = process_data.dig('process', 'dataSources')
+        unless data_sources.blank?
+          data_sources.map do |data_source|
+            get_data_source_alias(data_source['id'], client, aliases)
+          end
+        end
+        component = process_data.dig('process', 'component')
+        get_data_source_alias(component['configLocation']['dataSourceConfig']['id'], client, aliases) if component&.dig('configLocation', 'dataSourceConfig')
+        aliases
+      end
+
+      def get_data_source_alias(data_source_id, client, aliases)
+        unless aliases[data_source_id]
+          data_source = GoodData::Helpers.get_data_source_by_id(data_source_id, client)
+          if data_source&.dig('dataSource', 'alias')
+            aliases[data_source_id] = {
+              :type => get_data_source_type(data_source),
+              :alias => data_source['dataSource']['alias']
+            }
+          end
+        end
+        aliases[data_source_id]
+      end
+
+      def get_data_source_type(data_source_data)
+        data_source_data&.dig('dataSource', 'connectionInfo') ? data_source_data['dataSource']['connectionInfo'].first[0].upcase : ""
+      end
+
+      def replace_process_data_source_ids(process_data, client, aliases)
+        component = process_data.dig(:process, :component)
+        if component&.dig(:configLocation, :dataSourceConfig)
+          the_alias = aliases[component[:configLocation][:dataSourceConfig][:id]]
+          process_data[:process][:component][:configLocation][:dataSourceConfig][:id] = GoodData::Helpers.verify_data_source_alias(the_alias, client)
+        end
+        process_data[:process][:dataSources] = replace_data_source_ids(process_data[:process][:dataSources], client, aliases)
+        process_data
+      end
+
+      def replace_data_source_ids(data_sources, client, aliases)
+        array_data_sources = []
+        if data_sources && !data_sources.empty?
+          data_sources.map do |data_source|
+            new_id = GoodData::Helpers.verify_data_source_alias(aliases[data_source[:id]], client)
+            array_data_sources.push(:id => new_id)
+          end
+        end
+        array_data_sources
+      end
+
       def transfer_user_groups(from_project, to_project)
         from_project.user_groups.map do |ug|
           # migrate groups
@@ -625,6 +679,7 @@ module GoodData
     def blueprint(options = {})
       options = { include_ca: true }.merge(options)
       result = client.get("/gdc/projects/#{pid}/model/view", params: { includeDeprecated: true, includeGrain: true, includeCA: options[:include_ca] })
+
       polling_url = result['asyncTask']['link']['poll']
       model = client.poll_on_code(polling_url, options)
       bp = GoodData::Model::FromWire.from_wire(model, options)
@@ -1922,6 +1977,20 @@ module GoodData
       [user, roles]
     end
 
+    def upgrade_custom_v2(message, options = {})
+      uri = "/gdc/md/#{pid}/datedimension/upgrade"
+      poll_result = client&.post(uri, message)
+
+      return poll_result['wTaskStatus']['status'] if poll_result['wTaskStatus'] && poll_result['wTaskStatus']['status']
+
+      polling_uri = poll_result['asyncTask']['link']['poll']
+      result = client&.poll_on_response(polling_uri, options) do |body|
+        body && body['wTaskStatus'] && body['wTaskStatus']['status'] == 'RUNNING'
+      end
+
+      result['wTaskStatus']['status'] == 'OK' ? 'OK' : 'FAIL'
+    end
+
     def add
       @add ||= GoodData::AutomatedDataDistribution.new(self)
       @add

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gooddata
 version: !ruby/object:Gem::Version
-  version: 2.1.12
+  version: 2.1.13
 platform: ruby
 authors:
 - Pavel Kolesnikov
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-22 00:00:00.000000000 Z
+date: 2020-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: license_finder
@@ -757,6 +757,7 @@ files:
 - lib/gooddata/helpers/crypto_helper.rb
 - lib/gooddata/helpers/csv_helper.rb
 - lib/gooddata/helpers/data_helper.rb
+- lib/gooddata/helpers/data_source_helpers.rb
 - lib/gooddata/helpers/erb_helper.rb
 - lib/gooddata/helpers/global_helpers.rb
 - lib/gooddata/helpers/global_helpers_params.rb
@@ -785,6 +786,7 @@ files:
 - lib/gooddata/lcm/actions/hello_world.rb
 - lib/gooddata/lcm/actions/help.rb
 - lib/gooddata/lcm/actions/import_object_collections.rb
+- lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb
 - lib/gooddata/lcm/actions/provision_clients.rb
 - lib/gooddata/lcm/actions/purge_clients.rb
 - lib/gooddata/lcm/actions/rename_existing_client_projects.rb