gooddata 2.1.8-java → 2.1.13-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +7 -0
  3. data/.travis.yml +2 -4
  4. data/CHANGELOG.md +43 -0
  5. data/Dockerfile +19 -4
  6. data/Dockerfile.jruby +4 -4
  7. data/Dockerfile.ruby +5 -4
  8. data/README.md +2 -0
  9. data/SDK_VERSION +1 -1
  10. data/VERSION +1 -1
  11. data/bin/provision.sh +2 -0
  12. data/bin/release.sh +2 -0
  13. data/bin/rollout.sh +2 -0
  14. data/bin/run_brick.rb +31 -7
  15. data/bin/test_projects_cleanup.rb +10 -2
  16. data/bin/user_filters.sh +2 -0
  17. data/ci.rake +1 -1
  18. data/ci/bigquery/pom.xml +54 -0
  19. data/ci/redshift/pom.xml +73 -0
  20. data/ci/snowflake/pom.xml +57 -0
  21. data/dev-gooddata-sso.pub.encrypted +40 -40
  22. data/gdc_fossa_lcm.yaml +2 -0
  23. data/gdc_fossa_ruby_sdk.yaml +4 -0
  24. data/gooddata.gemspec +6 -2
  25. data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
  26. data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml +22 -12
  27. data/lcm.rake +14 -0
  28. data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +68 -0
  29. data/lib/gooddata/bricks/middleware/logger_middleware.rb +2 -1
  30. data/lib/gooddata/bricks/middleware/mask_logger_decorator.rb +5 -1
  31. data/lib/gooddata/bricks/pipeline.rb +7 -0
  32. data/lib/gooddata/cloud_resources/bigquery/bigquery_client.rb +86 -0
  33. data/lib/gooddata/cloud_resources/bigquery/drivers/.gitkeepme +0 -0
  34. data/lib/gooddata/cloud_resources/cloud_resouce_factory.rb +28 -0
  35. data/lib/gooddata/cloud_resources/cloud_resource_client.rb +24 -0
  36. data/lib/gooddata/cloud_resources/cloud_resources.rb +12 -0
  37. data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +15 -0
  38. data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +101 -0
  39. data/lib/gooddata/cloud_resources/snowflake/drivers/.gitkeepme +0 -0
  40. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +84 -0
  41. data/lib/gooddata/exceptions/invalid_env_error.rb +15 -0
  42. data/lib/gooddata/helpers/data_helper.rb +10 -0
  43. data/lib/gooddata/helpers/data_source_helpers.rb +47 -0
  44. data/lib/gooddata/helpers/global_helpers.rb +4 -0
  45. data/lib/gooddata/helpers/global_helpers_params.rb +6 -9
  46. data/lib/gooddata/lcm/actions/collect_clients.rb +6 -6
  47. data/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb +6 -6
  48. data/lib/gooddata/lcm/actions/collect_segment_clients.rb +4 -1
  49. data/lib/gooddata/lcm/actions/collect_segments.rb +1 -2
  50. data/lib/gooddata/lcm/actions/collect_users_brick_users.rb +7 -6
  51. data/lib/gooddata/lcm/actions/create_segment_masters.rb +5 -3
  52. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +116 -0
  53. data/lib/gooddata/lcm/actions/set_master_project.rb +76 -0
  54. data/lib/gooddata/lcm/actions/synchronize_clients.rb +1 -1
  55. data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +1 -2
  56. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +20 -3
  57. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +23 -3
  58. data/lib/gooddata/lcm/actions/synchronize_users.rb +50 -30
  59. data/lib/gooddata/lcm/actions/update_release_table.rb +7 -1
  60. data/lib/gooddata/lcm/exceptions/lcm_execution_error.rb +16 -0
  61. data/lib/gooddata/lcm/helpers/release_table_helper.rb +16 -8
  62. data/lib/gooddata/lcm/lcm2.rb +28 -5
  63. data/lib/gooddata/models/domain.rb +17 -15
  64. data/lib/gooddata/models/execution.rb +0 -1
  65. data/lib/gooddata/models/execution_detail.rb +0 -1
  66. data/lib/gooddata/models/from_wire.rb +1 -0
  67. data/lib/gooddata/models/process.rb +11 -3
  68. data/lib/gooddata/models/profile.rb +33 -11
  69. data/lib/gooddata/models/project.rb +120 -31
  70. data/lib/gooddata/models/project_creator.rb +2 -0
  71. data/lib/gooddata/models/schedule.rb +0 -1
  72. data/lib/gooddata/rest/client.rb +2 -2
  73. data/lib/gooddata/rest/connection.rb +5 -3
  74. data/rubydev_public.gpg.encrypted +51 -51
  75. data/rubydev_secret_keys.gpg.encrypted +109 -109
  76. metadata +32 -13
  77. data/lib/gooddata/extensions/hash.rb +0 -18
@@ -0,0 +1,28 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
4
+ # This source code is licensed under the BSD-style license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ require 'active_support/core_ext/string/inflections'
8
+ require_relative 'cloud_resource_client'
9
+
10
+ module GoodData
11
+ module CloudResources
12
+ class CloudResourceFactory
13
+ class << self
14
+ def create(type, data = {}, opts = {})
15
+ clients = CloudResourceClient.descendants.select { |c| c.respond_to?("accept?") && c.send("accept?", type) }
16
+ raise "DataSource does not support type \"#{type}\"" if clients.empty?
17
+
18
+ res = clients[0].new(data)
19
+ opts.each do |key, value|
20
+ method = "#{key}="
21
+ res.send(method, value) if res.respond_to?(method)
22
+ end
23
+ res
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,24 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
4
+ # This source code is licensed under the BSD-style license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ module GoodData
8
+ module CloudResources
9
+ class CloudResourceClient
10
+ def self.inherited(klass)
11
+ @descendants ||= []
12
+ @descendants << klass
13
+ end
14
+
15
+ def self.descendants
16
+ @descendants || []
17
+ end
18
+
19
+ def realize_query(_query, _params)
20
+ raise NotImplementedError, 'Must be implemented in subclass'
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,12 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
4
+ # This source code is licensed under the BSD-style license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ require 'pathname'
8
+
9
+ base = Pathname(__FILE__).dirname.expand_path
10
+ Dir.glob(base + '**/*.rb').each do |file|
11
+ require file
12
+ end
@@ -0,0 +1,15 @@
1
+ #
2
+ # Copyright (C) 2007-2019, GoodData(R) Corporation. All rights reserved.
3
+ #
4
+
5
+ #=======================================================================================================================
6
+ # Root Logger
7
+ #=======================================================================================================================
8
+ #log4j.rootCategory=INFO, Syslog, Console
9
+ log4j.rootCategory=INFO
10
+
11
+ #=======================================================================================================================
12
+ # Logger with Higher Verbosity
13
+ #=======================================================================================================================
14
+ log4j.logger.com.amazonaws=INFO
15
+
@@ -0,0 +1,101 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-redshift-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class RedshiftClient < CloudResourceClient
21
+ class << self
22
+ def accept?(type)
23
+ type == 'redshift'
24
+ end
25
+ end
26
+
27
+ def initialize(options = {})
28
+ raise("Data Source needs a client to Redshift to be able to query the storage but 'redshift_client' is empty.") unless options['redshift_client']
29
+
30
+ if options['redshift_client']['connection'].is_a?(Hash)
31
+ @database = options['redshift_client']['connection']['database']
32
+ @schema = options['redshift_client']['connection']['schema'] || 'public'
33
+ @url = options['redshift_client']['connection']['url']
34
+ @authentication = options['redshift_client']['connection']['authentication']
35
+ else
36
+ raise('Missing connection info for Redshift client')
37
+
38
+ end
39
+ @debug = options['debug'] == true || options['debug'] == 'true'
40
+
41
+ Java.com.amazon.redshift.jdbc42.Driver
42
+ base = Pathname(__FILE__).dirname
43
+ org.apache.log4j.PropertyConfigurator.configure("#{base}/drivers/log4j.properties")
44
+ end
45
+
46
+ def realize_query(query, _params)
47
+ GoodData.gd_logger.info("Realize SQL query: type=redshift status=started")
48
+
49
+ connect
50
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
51
+ measure = Benchmark.measure do
52
+ statement = @connection.create_statement
53
+ schema_sql = "set search_path to #{@schema}"
54
+ statement.execute(schema_sql)
55
+
56
+ has_result = statement.execute(query)
57
+ if has_result
58
+ result = statement.get_result_set
59
+ metadata = result.get_meta_data
60
+ col_count = metadata.column_count
61
+ CSV.open(filename, 'wb') do |csv|
62
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
63
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
64
+ end
65
+ end
66
+ end
67
+ GoodData.gd_logger.info("Realize SQL query: type=redshift status=finished duration=#{measure.real}")
68
+ filename
69
+ ensure
70
+ @connection.close unless @connection.nil?
71
+ @connection = nil
72
+ end
73
+
74
+ def connect
75
+ full_url = build_url(@url, @database)
76
+ GoodData.logger.info "Setting up connection to Redshift #{full_url}"
77
+
78
+ prop = java.util.Properties.new
79
+ if @authentication['basic']
80
+ prop.setProperty('UID', @authentication['basic']['userName'])
81
+ prop.setProperty('PWD', @authentication['basic']['password'])
82
+ else
83
+ prop.setProperty('AccessKeyID', @authentication['iam']['accessKeyId'])
84
+ prop.setProperty('SecretAccessKey', @authentication['iam']['secretAccessKey'])
85
+ prop.setProperty('DbUser', @authentication['iam']['dbUser'])
86
+ end
87
+
88
+ @connection = java.sql.DriverManager.getConnection(full_url, prop)
89
+ end
90
+
91
+ private
92
+
93
+ def build_url(url, database)
94
+ url_parts = url.split('?')
95
+ url_path = url_parts[0].chomp('/')
96
+ url_path += "/#{database}" if database && !url_path.end_with?("/#{database}")
97
+ url_parts.length > 1 ? url_path + '?' + url_parts[1] : url_path
98
+ end
99
+ end
100
+ end
101
+ end
@@ -0,0 +1,84 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-snowflake-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class SnowflakeClient < CloudResourceClient
21
+ class << self
22
+ def accept?(type)
23
+ type == 'snowflake'
24
+ end
25
+ end
26
+
27
+ def initialize(options = {})
28
+ raise("Data Source needs a client to Snowflake to be able to query the storage but 'snowflake_client' is empty.") unless options['snowflake_client']
29
+
30
+ if options['snowflake_client']['connection'].is_a?(Hash)
31
+ @database = options['snowflake_client']['connection']['database']
32
+ @schema = options['snowflake_client']['connection']['schema'] || 'public'
33
+ @warehouse = options['snowflake_client']['connection']['warehouse']
34
+ @url = options['snowflake_client']['connection']['url']
35
+ @authentication = options['snowflake_client']['connection']['authentication']
36
+ else
37
+ raise('Missing connection info for Snowflake client')
38
+
39
+ end
40
+
41
+ Java.net.snowflake.client.jdbc.SnowflakeDriver
42
+ end
43
+
44
+ def realize_query(query, _params)
45
+ GoodData.gd_logger.info("Realize SQL query: type=snowflake status=started")
46
+
47
+ connect
48
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
49
+ measure = Benchmark.measure do
50
+ statement = @connection.create_statement
51
+
52
+ has_result = statement.execute(query)
53
+ if has_result
54
+ result = statement.get_result_set
55
+ metadata = result.get_meta_data
56
+ col_count = metadata.column_count
57
+ CSV.open(filename, 'wb') do |csv|
58
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
59
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
60
+ end
61
+ end
62
+ end
63
+ GoodData.gd_logger.info("Realize SQL query: type=snowflake status=finished duration=#{measure.real}")
64
+ filename
65
+ ensure
66
+ @connection&.close
67
+ @connection = nil
68
+ end
69
+
70
+ def connect
71
+ GoodData.logger.info "Setting up connection to Snowflake #{@url}"
72
+
73
+ prop = java.util.Properties.new
74
+ prop.setProperty('user', @authentication['basic']['userName'])
75
+ prop.setProperty('password', @authentication['basic']['password'])
76
+ prop.setProperty('schema', @schema)
77
+ prop.setProperty('warehouse', @warehouse)
78
+ prop.setProperty('db', @database)
79
+
80
+ @connection = java.sql.DriverManager.getConnection(@url, prop)
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,15 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
4
+ # This source code is licensed under the BSD-style license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ module GoodData
8
+ class InvalidEnvError < RuntimeError
9
+ DEFAULT_MSG = 'Invalid environment: It must be JAVA platform'
10
+
11
+ def initialize(msg = DEFAULT_MSG)
12
+ super(msg)
13
+ end
14
+ end
15
+ end
@@ -44,6 +44,11 @@ module GoodData
44
44
  realize_link
45
45
  when 's3'
46
46
  realize_s3(params)
47
+ when 'redshift', 'snowflake', 'bigquery'
48
+ raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
49
+
50
+ require_relative '../cloud_resources/cloud_resources'
51
+ realize_cloud_resource(source, params)
47
52
  else
48
53
  raise "DataSource does not support type \"#{source}\""
49
54
  end
@@ -55,6 +60,11 @@ module GoodData
55
60
 
56
61
  private
57
62
 
63
+ def realize_cloud_resource(type, params)
64
+ cloud_resource_client = GoodData::CloudResources::CloudResourceFactory.create(type, params)
65
+ cloud_resource_client.realize_query(@options[:query], params)
66
+ end
67
+
58
68
  def realize_query(params)
59
69
  query = DataSource.interpolate_sql_params(@options[:query], params)
60
70
  dwh = params['ads_client'] || params[:ads_client] || raise("Data Source needs a client to ads to be able to query the storage but 'ads_client' is empty.")
@@ -0,0 +1,47 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2020 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+ # frozen_string_literal: false
8
+
9
+ module GoodData
10
+ module Helpers
11
+ class << self
12
+ # Get a data source information from server by id
13
+ #
14
+ # @param [String] data_source_id The data source ID
15
+ # @param [Object] client The Rest Client object
16
+ # @return [Hash] Returns Data source
17
+ def get_data_source_by_id(data_source_id, client)
18
+ unless data_source_id.blank?
19
+ uri = "/gdc/dataload/dataSources/#{data_source_id}"
20
+ client.get(uri)
21
+ end
22
+ end
23
+
24
+ # Verify to see if the data source exists in the domain using its alias
25
+ #
26
+ # @param [String] ds_alias The data source's alias
27
+ # @param [Object] client The Rest Client object
28
+ # @return [String] Id of the data source or failed with the reason
29
+ def verify_data_source_alias(ds_alias, client)
30
+ domain = client.connection.server.url
31
+ fail "The data source alias is empty, check your data source configuration." unless ds_alias
32
+
33
+ uri = "/gdc/dataload/dataSources/internal/availableAlias?alias=#{ds_alias[:alias]}"
34
+ res = client.get(uri)
35
+ fail "Unable to get information about the Data Source '#{ds_alias[:alias]}' in the domain '#{domain}'" unless res
36
+ fail "Unable to find the #{ds_alias[:type]} Data Source '#{ds_alias[:alias]}' in the domain '#{domain}'" if res['availableAlias']['available']
37
+
38
+ ds_type = res['availableAlias']['existingDataSource']['type']
39
+ if ds_type && ds_type != ds_alias[:type]
40
+ fail "Wrong Data Source type - the '#{ds_type}' type is expected but the Data Source '#{ds_alias[:alias]}' in the domain '#{domain}' has the '#{ds_alias[:type]}' type"
41
+ else
42
+ res['availableAlias']['existingDataSource']['id']
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -183,6 +183,10 @@ module GoodData
183
183
  end
184
184
  end
185
185
 
186
+ def deep_merge(source, target)
187
+ GoodData::Helpers::DeepMergeableHash[source].deep_merge(target)
188
+ end
189
+
186
190
  def undot(params)
187
191
  # for each key-value config given
188
192
  params.map do |k, v|
@@ -3,10 +3,6 @@
3
3
  # LICENSE file in the root directory of this source tree.
4
4
  require 'active_support/core_ext/hash/slice'
5
5
 
6
- require 'gooddata/extensions/hash'
7
-
8
- using HashExtensions
9
-
10
6
  module GoodData
11
7
  module Helpers
12
8
  ENCODED_PARAMS_KEY = 'gd_encoded_params'
@@ -102,7 +98,8 @@ module GoodData
102
98
 
103
99
  params.delete(key)
104
100
  params.delete(hidden_key)
105
- params = params.deep_merge(parsed_data_params).deep_merge(parsed_hidden_data_params)
101
+ params = GoodData::Helpers.deep_merge(params, parsed_data_params)
102
+ params = GoodData::Helpers.deep_merge(params, parsed_hidden_data_params)
106
103
 
107
104
  if options[:convert_pipe_delimited_params]
108
105
  convert_pipe_delimited_params = lambda do |args|
@@ -121,7 +118,7 @@ module GoodData
121
118
  end
122
119
 
123
120
  lines.reduce({}) do |a, e|
124
- a.deep_merge(e)
121
+ GoodData::Helpers.deep_merge(a, e)
125
122
  end
126
123
  end
127
124
 
@@ -129,7 +126,7 @@ module GoodData
129
126
  params.delete_if do |k, _|
130
127
  k.include?('|')
131
128
  end
132
- params = params.deep_merge(pipe_delimited_params)
129
+ params = GoodData::Helpers.deep_merge(params, pipe_delimited_params)
133
130
  end
134
131
 
135
132
  params
@@ -245,7 +242,7 @@ module GoodData
245
242
 
246
243
  def resolve_reference_params(data_params, params)
247
244
  reference_values = []
248
- regexps = Regexp.union(/\\\\/, /\\\$/, /\$\{(\w+)\}/)
245
+ regexps = Regexp.union(/\\\\/, /\\\$/, /\$\{([\w\s\.]+)\}/)
249
246
  resolve_reference = lambda do |v|
250
247
  if v.is_a? Hash
251
248
  Hash[
@@ -265,7 +262,7 @@ module GoodData
265
262
  data_params.is_a?(Hash) ? '\\' : '\\\\' # rubocop: disable Metrics/BlockNesting
266
263
  elsif match =~ /\\\$/
267
264
  '$'
268
- elsif match =~ /\$\{(\w+)\}/
265
+ elsif match =~ /\$\{([\w\s\.]+)\}/
269
266
  val = params["#{$1}"]
270
267
  if val
271
268
  reference_values << val
@@ -67,11 +67,11 @@ module GoodData
67
67
  end
68
68
 
69
69
  def collect_clients(params, segment_names = nil)
70
- client_id_column = params.client_id_column || 'client_id'
71
- segment_id_column = params.segment_id_column || 'segment_id'
72
- project_id_column = params.project_id_column || 'project_id'
73
- project_title_column = params.project_title_column || 'project_title'
74
- project_token_column = params.project_token_column || 'project_token'
70
+ client_id_column = params.client_id_column&.downcase || 'client_id'
71
+ segment_id_column = params.segment_id_column&.downcase || 'segment_id'
72
+ project_id_column = params.project_id_column&.downcase || 'project_id'
73
+ project_title_column = params.project_title_column&.downcase || 'project_title'
74
+ project_token_column = params.project_token_column&.downcase || 'project_token'
75
75
  client = params.gdc_gd_client
76
76
 
77
77
  clients = []
@@ -82,7 +82,7 @@ module GoodData
82
82
  end
83
83
  GoodData.logger.debug("Input data: #{input_data.read}")
84
84
  GoodData.logger.debug("Segment names: #{segment_names}")
85
- CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
85
+ CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
86
86
  GoodData.logger.debug("Processing row: #{row}")
87
87
  segment_name = row[segment_id_column]
88
88
  GoodData.logger.debug("Segment name: #{segment_name}")