gooddata 2.1.14 → 2.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (78) hide show
  1. checksums.yaml +5 -5
  2. data/.gdc-ii-config.yaml +1 -1
  3. data/.github/workflows/build.yml +66 -0
  4. data/.github/workflows/pre-merge.yml +72 -0
  5. data/.sonar.settings +4 -0
  6. data/.travis.yml +78 -12
  7. data/CHANGELOG.md +62 -0
  8. data/Dockerfile +25 -14
  9. data/LICENSE +4418 -17
  10. data/LICENSE.rb +1 -1
  11. data/README.md +3 -3
  12. data/Rakefile +8 -1
  13. data/SDK_VERSION +1 -1
  14. data/VERSION +1 -1
  15. data/bin/test_projects_cleanup.rb +45 -3
  16. data/ci/mssql/pom.xml +62 -0
  17. data/ci/mysql/pom.xml +57 -0
  18. data/ci/postgresql/pom.xml +57 -0
  19. data/ci/redshift/pom.xml +1 -1
  20. data/dev-gooddata-sso.pub.encrypted +40 -40
  21. data/gdc_fossa_ruby_sdk.yaml +1 -0
  22. data/gooddata.gemspec +10 -6
  23. data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
  24. data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml +11 -1
  25. data/lcm.rake +2 -2
  26. data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
  27. data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
  28. data/lib/gooddata/cloud_resources/{cloud_resouce_factory.rb → cloud_resource_factory.rb} +8 -0
  29. data/lib/gooddata/cloud_resources/cloud_resources.rb +1 -1
  30. data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
  31. data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
  32. data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
  33. data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
  34. data/lib/gooddata/cloud_resources/postgresql/drivers/.gitkeepme +0 -0
  35. data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +106 -0
  36. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
  37. data/lib/gooddata/commands/scaffold.rb +9 -10
  38. data/lib/gooddata/core/nil_logger.rb +3 -1
  39. data/lib/gooddata/helpers/data_helper.rb +9 -5
  40. data/lib/gooddata/helpers/global_helpers.rb +6 -5
  41. data/lib/gooddata/lcm/actions/associate_clients.rb +8 -2
  42. data/lib/gooddata/lcm/actions/base_action.rb +0 -2
  43. data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
  44. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
  45. data/lib/gooddata/lcm/actions/provision_clients.rb +31 -10
  46. data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
  47. data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
  48. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
  49. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +16 -9
  50. data/lib/gooddata/lcm/actions/synchronize_users.rb +7 -6
  51. data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
  52. data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
  53. data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
  54. data/lib/gooddata/lcm/lcm2.rb +5 -2
  55. data/lib/gooddata/lcm/types/base_type.rb +0 -2
  56. data/lib/gooddata/mixins/md_object_query.rb +9 -6
  57. data/lib/gooddata/models/blueprint/project_blueprint.rb +0 -2
  58. data/lib/gooddata/models/client.rb +14 -12
  59. data/lib/gooddata/models/data_source.rb +668 -0
  60. data/lib/gooddata/models/dataset_mapping.rb +36 -0
  61. data/lib/gooddata/models/domain.rb +3 -2
  62. data/lib/gooddata/models/metadata/analytical_dashboard.rb +49 -0
  63. data/lib/gooddata/models/metadata/analytical_visualization_object.rb +30 -0
  64. data/lib/gooddata/models/metadata/label.rb +26 -27
  65. data/lib/gooddata/models/metadata/visualization_object.rb +50 -0
  66. data/lib/gooddata/models/project.rb +66 -19
  67. data/lib/gooddata/models/schedule.rb +13 -1
  68. data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -54
  69. data/lib/gooddata/models/user_group.rb +0 -1
  70. data/lib/gooddata/rest/connection.rb +6 -4
  71. data/lib/gooddata/rest/phmap.rb +2 -1
  72. data/lib/gooddata.rb +2 -0
  73. data/rubydev_public.gpg.encrypted +51 -51
  74. data/rubydev_secret_keys.gpg.encrypted +109 -109
  75. metadata +52 -27
  76. data/DEPENDENCIES.md +0 -880
  77. data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
  78. data/lib/gooddata/helpers/data_source_helpers.rb +0 -47
@@ -0,0 +1,98 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'pathname'
10
+ require "azure/storage/blob"
11
+
12
+ module GoodData
13
+ class BlobStorageClient
14
+ SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
15
+ INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
16
+ INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
17
+ INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
18
+ INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
19
+ INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
20
+ INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
21
+ INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
22
+ INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
23
+ INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
24
+
25
+ attr_reader :use_sas
26
+
27
+ def initialize(options = {})
28
+ raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
29
+
30
+ if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
31
+ @connection_string = options['blobStorage_client']['connectionString']
32
+ @container = options['blobStorage_client']['container']
33
+ @path = options['blobStorage_client']['path']
34
+ @use_sas = false
35
+ build_sas(@connection_string)
36
+ else
37
+ raise('Missing connection info for Blob Storage client')
38
+ end
39
+ end
40
+
41
+ def realize_blob(file, _params)
42
+ GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
43
+ filename = ''
44
+ begin
45
+ connect
46
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
47
+ blob_name = get_blob_name(@path, file)
48
+
49
+ measure = Benchmark.measure do
50
+ _blob, content = @client.get_blob(@container, blob_name)
51
+ File.open(filename, "wb") { |f| f.write(content) }
52
+ end
53
+ rescue => e
54
+ raise_error(e)
55
+ end
56
+ GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
57
+ filename
58
+ end
59
+
60
+ def connect
61
+ GoodData.logger.info "Setting up connection to Blob Storage"
62
+ if use_sas
63
+ @client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
64
+ else
65
+ @client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
66
+ end
67
+ end
68
+
69
+ def build_sas(url)
70
+ matches = url.scan(SAS_URL_PATTERN)
71
+ return unless matches && matches[0]
72
+
73
+ @use_sas = true
74
+ @host = matches[0][0]
75
+ @sas_token = matches[0][1]
76
+ end
77
+
78
+ def raise_error(e)
79
+ if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
80
+ raise INVALID_BLOB_EXPIRED_MESSAGE
81
+ elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
82
+ raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
83
+ elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
84
+ raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
85
+ elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
86
+ raise INVALID_BLOB_INVALID_PATH_MESSAGE
87
+ else
88
+ raise INVALID_BLOB_GENERAL_MESSAGE
89
+ end
90
+ end
91
+
92
+ def get_blob_name(path, file)
93
+ return file unless path
94
+
95
+ path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
96
+ end
97
+ end
98
+ end
@@ -11,7 +11,15 @@ module GoodData
11
11
  module CloudResources
12
12
  class CloudResourceFactory
13
13
  class << self
14
+ def load_cloud_resource(type)
15
+ base = "#{Pathname(__FILE__).dirname.expand_path}#{File::SEPARATOR}#{type}#{File::SEPARATOR}"
16
+ Dir.glob(base + '**/*.rb').each do |file|
17
+ require file
18
+ end
19
+ end
20
+
14
21
  def create(type, data = {}, opts = {})
22
+ load_cloud_resource(type)
15
23
  clients = CloudResourceClient.descendants.select { |c| c.respond_to?("accept?") && c.send("accept?", type) }
16
24
  raise "DataSource does not support type \"#{type}\"" if clients.empty?
17
25
 
@@ -7,6 +7,6 @@
7
7
  require 'pathname'
8
8
 
9
9
  base = Pathname(__FILE__).dirname.expand_path
10
- Dir.glob(base + '**/*.rb').each do |file|
10
+ Dir.glob(base + '*.rb').each do |file|
11
11
  require file
12
12
  end
@@ -0,0 +1,122 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mssql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MSSQLClient < CloudResourceClient
21
+ MSSQL_SEPARATOR_PARAM = ";"
22
+ MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
23
+ MSSQL_DEFAULT_PORT = 1433
24
+ LOGIN_TIME_OUT = 30
25
+ MSSQL_FETCH_SIZE = 10_000
26
+ VERIFY_FULL = 'verify-full'
27
+ PREFER = 'prefer'
28
+ REQUIRE = 'require'
29
+
30
+ class << self
31
+ def accept?(type)
32
+ type == 'mssql'
33
+ end
34
+ end
35
+
36
+ def initialize(options = {})
37
+ raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
38
+
39
+ connection = options['mssql_client']['connection']
40
+ if connection.is_a?(Hash)
41
+ @database = connection['database']
42
+ @schema = connection['schema']
43
+ @authentication = connection['authentication']
44
+ @ssl_mode = connection['sslMode']
45
+ @url = connection['url']
46
+
47
+ validate
48
+ else
49
+ raise('Missing connection info for MSSQL client')
50
+ end
51
+
52
+ Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
53
+ end
54
+
55
+ def realize_query(query, _params)
56
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
57
+
58
+ connect
59
+
60
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
61
+ measure = Benchmark.measure do
62
+ statement = @connection.create_statement
63
+ statement.set_fetch_size(MSSQL_FETCH_SIZE)
64
+ has_result = statement.execute(query)
65
+ if has_result
66
+ result = statement.get_result_set
67
+ metadata = result.get_meta_data
68
+ col_count = metadata.column_count
69
+ CSV.open(filename, 'wb') do |csv|
70
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
71
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
72
+ end
73
+ end
74
+ end
75
+
76
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
77
+ filename
78
+ ensure
79
+ @connection&.close
80
+ @connection = nil
81
+ end
82
+
83
+ def connect
84
+ connection_string = build_connection_string
85
+ GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
86
+
87
+ authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
88
+
89
+ prop = java.util.Properties.new
90
+ prop.setProperty('userName', authentication['userName'])
91
+ prop.setProperty('password', authentication['password'])
92
+
93
+ @connection = java.sql.DriverManager.getConnection(connection_string, prop)
94
+ end
95
+
96
+ def validate
97
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
98
+
99
+ raise "Instance name is not supported" if @url !~ /^[^\\]*$/
100
+
101
+ raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
102
+
103
+ url_matches = @url.scan(MSSQL_URL_PATTERN)
104
+ raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
105
+
106
+ raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
107
+ end
108
+
109
+ def build_connection_string
110
+ encrypt = @ssl_mode != PREFER
111
+ trust_server_certificate = @ssl_mode == REQUIRE
112
+
113
+ "#{@url};" \
114
+ "database=#{@database};" \
115
+ "encrypt=#{encrypt};" \
116
+ "trustServerCertificate=#{trust_server_certificate};" \
117
+ "loginTimeout=#{LOGIN_TIME_OUT};" \
118
+ "#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,111 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mysql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MysqlClient < CloudResourceClient
21
+ JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
22
+ MYSQL_DEFAULT_PORT = 3306
23
+ JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
24
+ VERIFY_FULL = 'VERIFY_IDENTITY'
25
+ PREFER = 'PREFERRED'
26
+ REQUIRE = 'REQUIRED'
27
+ MYSQL_FETCH_SIZE = 1000
28
+
29
+ class << self
30
+ def accept?(type)
31
+ type == 'mysql'
32
+ end
33
+ end
34
+
35
+ def initialize(options = {})
36
+ raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
37
+
38
+ if options['mysql_client']['connection'].is_a?(Hash)
39
+ @database = options['mysql_client']['connection']['database']
40
+ @authentication = options['mysql_client']['connection']['authentication']
41
+ @ssl_mode = options['mysql_client']['connection']['sslMode']
42
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
43
+
44
+ @url = build_url(options['mysql_client']['connection']['url'])
45
+ else
46
+ raise('Missing connection info for Mysql client')
47
+ end
48
+
49
+ Java.com.mysql.cj.jdbc.Driver
50
+ end
51
+
52
+ def realize_query(query, _params)
53
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
54
+
55
+ connect
56
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
57
+ measure = Benchmark.measure do
58
+ statement = @connection.create_statement
59
+ statement.set_fetch_size(MYSQL_FETCH_SIZE)
60
+ has_result = statement.execute(query)
61
+ if has_result
62
+ result = statement.get_result_set
63
+ metadata = result.get_meta_data
64
+ col_count = metadata.column_count
65
+ CSV.open(filename, 'wb') do |csv|
66
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
67
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
68
+ end
69
+ end
70
+ end
71
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
72
+ filename
73
+ ensure
74
+ @connection&.close
75
+ @connection = nil
76
+ end
77
+
78
+ def connect
79
+ GoodData.logger.info "Setting up connection to Mysql #{@url}"
80
+
81
+ prop = java.util.Properties.new
82
+ prop.setProperty('user', @authentication['basic']['userName'])
83
+ prop.setProperty('password', @authentication['basic']['password'])
84
+
85
+ @connection = java.sql.DriverManager.getConnection(@url, prop)
86
+ @connection.set_auto_commit(false)
87
+ end
88
+
89
+ def build_url(url)
90
+ matches = url.scan(JDBC_MYSQL_PATTERN)
91
+ raise 'Cannot reach the url' unless matches
92
+
93
+ host = matches[0][0]
94
+ port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
95
+
96
+ "#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
97
+ end
98
+
99
+ def get_ssl_mode(ssl_mode)
100
+ mode = PREFER
101
+ if ssl_mode == 'verify-full'
102
+ mode = VERIFY_FULL
103
+ elsif ssl_mode == 'require'
104
+ mode = REQUIRE
105
+ end
106
+
107
+ mode
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,106 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-postgresql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class PostgresClient < CloudResourceClient
21
+ JDBC_POSTGRES_PATTERN = %r{jdbc:postgresql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
22
+ POSTGRES_DEFAULT_PORT = 5432
23
+ JDBC_POSTGRES_PROTOCOL = 'jdbc:postgresql://'
24
+ SSL_JAVA_FACTORY = '&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory'
25
+ VERIFY_FULL = 'verify-full'
26
+ PREFER = 'prefer'
27
+ REQUIRE = 'require'
28
+ POSTGRES_SET_SCHEMA_COMMAND = "set search_path to"
29
+ POSTGRES_FETCH_SIZE = 1000
30
+
31
+ class << self
32
+ def accept?(type)
33
+ type == 'postgresql'
34
+ end
35
+ end
36
+
37
+ def initialize(options = {})
38
+ raise("Data Source needs a client to Postgres to be able to query the storage but 'postgresql_client' is empty.") unless options['postgresql_client']
39
+
40
+ if options['postgresql_client']['connection'].is_a?(Hash)
41
+ @database = options['postgresql_client']['connection']['database']
42
+ @schema = options['postgresql_client']['connection']['schema'] || 'public'
43
+ @authentication = options['postgresql_client']['connection']['authentication']
44
+ @ssl_mode = options['postgresql_client']['connection']['sslMode']
45
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
46
+
47
+ @url = build_url(options['postgresql_client']['connection']['url'])
48
+ else
49
+ raise('Missing connection info for Postgres client')
50
+ end
51
+
52
+ Java.org.postgresql.Driver
53
+ end
54
+
55
+ def realize_query(query, _params)
56
+ GoodData.gd_logger.info("Realize SQL query: type=postgresql status=started")
57
+
58
+ connect
59
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
60
+ measure = Benchmark.measure do
61
+ statement = @connection.create_statement
62
+ statement.set_fetch_size(POSTGRES_FETCH_SIZE)
63
+ has_result = statement.execute(query)
64
+ if has_result
65
+ result = statement.get_result_set
66
+ metadata = result.get_meta_data
67
+ col_count = metadata.column_count
68
+ CSV.open(filename, 'wb') do |csv|
69
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
70
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
71
+ end
72
+ end
73
+ end
74
+ GoodData.gd_logger.info("Realize SQL query: type=postgresql status=finished duration=#{measure.real}")
75
+ filename
76
+ ensure
77
+ @connection&.close
78
+ @connection = nil
79
+ end
80
+
81
+ def connect
82
+ GoodData.logger.info "Setting up connection to Postgresql #{@url}"
83
+
84
+ prop = java.util.Properties.new
85
+ prop.setProperty('user', @authentication['basic']['userName'])
86
+ prop.setProperty('password', @authentication['basic']['password'])
87
+ prop.setProperty('schema', @schema)
88
+
89
+ @connection = java.sql.DriverManager.getConnection(@url, prop)
90
+ statement = @connection.create_statement
91
+ statement.execute("#{POSTGRES_SET_SCHEMA_COMMAND} #{@schema}")
92
+ @connection.set_auto_commit(false)
93
+ end
94
+
95
+ def build_url(url)
96
+ matches = url.scan(JDBC_POSTGRES_PATTERN)
97
+ raise 'Cannot reach the url' unless matches
98
+
99
+ host = matches[0][0]
100
+ port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
101
+
102
+ "#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
103
+ end
104
+ end
105
+ end
106
+ end
@@ -18,6 +18,9 @@ end
18
18
  module GoodData
19
19
  module CloudResources
20
20
  class SnowflakeClient < CloudResourceClient
21
+ SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
22
+ SNOWFLAKE_SEPARATOR_PARAM = '?'
23
+
21
24
  class << self
22
25
  def accept?(type)
23
26
  type == 'snowflake'
@@ -31,7 +34,7 @@ module GoodData
31
34
  @database = options['snowflake_client']['connection']['database']
32
35
  @schema = options['snowflake_client']['connection']['schema'] || 'public'
33
36
  @warehouse = options['snowflake_client']['connection']['warehouse']
34
- @url = options['snowflake_client']['connection']['url']
37
+ @url = build_url(options['snowflake_client']['connection']['url'])
35
38
  @authentication = options['snowflake_client']['connection']['authentication']
36
39
  else
37
40
  raise('Missing connection info for Snowflake client')
@@ -79,6 +82,20 @@ module GoodData
79
82
 
80
83
  @connection = java.sql.DriverManager.getConnection(@url, prop)
81
84
  end
85
+
86
+ def build_url(url)
87
+ is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
88
+ unless is_contain
89
+ if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
90
+ url.concat("&")
91
+ else
92
+ url.concat(SNOWFLAKE_SEPARATOR_PARAM)
93
+ end
94
+ url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
95
+ end
96
+
97
+ url
98
+ end
82
99
  end
83
100
  end
84
101
  end
@@ -4,7 +4,7 @@
4
4
  # This source code is licensed under the BSD-style license found in the
5
5
  # LICENSE file in the root directory of this source tree.
6
6
 
7
- require 'erubis'
7
+ require 'erb'
8
8
  require 'fileutils'
9
9
  require 'pathname'
10
10
 
@@ -18,15 +18,14 @@ module GoodData
18
18
  # TODO: Add option for custom output dir
19
19
  def project(name)
20
20
  fail ArgumentError, 'No name specified' if name.nil?
21
-
22
21
  FileUtils.mkdir(name)
23
22
  FileUtils.cd(name) do
24
23
  FileUtils.mkdir('model')
25
24
  FileUtils.cd('model') do
26
25
  input = File.read(TEMPLATES_PATH + 'project/model/model.rb.erb')
27
- eruby = Erubis::Eruby.new(input)
26
+ erb = ERB.new(input)
28
27
  File.open('model.rb', 'w') do |f|
29
- f.write(eruby.result(:name => name))
28
+ f.write(erb.result_with_hash(:name => name))
30
29
  end
31
30
  end
32
31
 
@@ -36,9 +35,9 @@ module GoodData
36
35
  end
37
36
 
38
37
  input = File.read(TEMPLATES_PATH + 'project/Goodfile.erb')
39
- eruby = Erubis::Eruby.new(input)
38
+ erb = ERB.new(input)
40
39
  File.open('Goodfile', 'w') do |f|
41
- f.write(eruby.result)
40
+ f.write(erb.result)
42
41
  end
43
42
  end
44
43
  end
@@ -51,15 +50,15 @@ module GoodData
51
50
  FileUtils.mkdir(name)
52
51
  FileUtils.cd(name) do
53
52
  input = File.read(TEMPLATES_PATH + 'bricks/brick.rb.erb')
54
- eruby = Erubis::Eruby.new(input)
53
+ erb = ERB.new(input)
55
54
  File.open('brick.rb', 'w') do |f|
56
- f.write(eruby.result)
55
+ f.write(erb.result)
57
56
  end
58
57
 
59
58
  input = File.read(TEMPLATES_PATH + 'bricks/main.rb.erb')
60
- eruby = Erubis::Eruby.new(input)
59
+ erb = ERB.new(input)
61
60
  File.open('main.rb', 'w') do |f|
62
- f.write(eruby.result)
61
+ f.write(erb.result)
63
62
  end
64
63
  end
65
64
  end
@@ -1,4 +1,5 @@
1
- # Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
1
+ # frozen_string_literal: true
2
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
2
3
  # This source code is licensed under the BSD-style license found in the
3
4
  # LICENSE file in the root directory of this source tree.
4
5
 
@@ -11,6 +12,7 @@ module GoodData
11
12
  @level = nil
12
13
  end
13
14
 
15
+ # No body define need for dummy logger
14
16
  def debug(*_args)
15
17
  end
16
18
 
@@ -44,11 +44,14 @@ module GoodData
44
44
  realize_link
45
45
  when 's3'
46
46
  realize_s3(params)
47
- when 'redshift', 'snowflake', 'bigquery'
47
+ when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
48
48
  raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
49
-
50
49
  require_relative '../cloud_resources/cloud_resources'
51
50
  realize_cloud_resource(source, params)
51
+ when 'blobStorage'
52
+ require_relative '../cloud_resources/blobstorage/blobstorage_client'
53
+ blob_storage_client = GoodData::BlobStorageClient.new(params)
54
+ blob_storage_client.realize_blob(@options[:file], params)
52
55
  else
53
56
  raise "DataSource does not support type \"#{source}\""
54
57
  end
@@ -112,12 +115,13 @@ module GoodData
112
115
  end
113
116
 
114
117
  def realize_s3(params)
115
- s3_client = params['aws_client'] && params['aws_client']['s3_client']
118
+ s3_client = params['s3_client'] && params['s3_client']['client']
116
119
  raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
117
120
  bucket_name = @options[:bucket]
118
- key = @options[:key]
121
+ key = @options[:key].present? ? @options[:key] : @options[:file]
119
122
  raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
120
- raise 'Key "key" is missing in S3 datasource' if key.blank?
123
+ raise 'Key "key" or "file" is missing in S3 datasource' if key.blank?
124
+
121
125
  GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
122
126
  filename = Digest::SHA256.new.hexdigest(@options.to_json)
123
127
  bucket = s3_client.bucket(bucket_name)
@@ -1,6 +1,6 @@
1
1
  # encoding: UTF-8
2
2
  #
3
- # Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
3
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
4
4
  # This source code is licensed under the BSD-style license found in the
5
5
  # LICENSE file in the root directory of this source tree.
6
6
 
@@ -34,6 +34,7 @@ module GoodData
34
34
  end
35
35
 
36
36
  set_const :GD_MAX_RETRY, (ENV['GD_MAX_RETRY'] && ENV['GD_MAX_RETRY'].to_i) || 12
37
+ AES_256_CBC_CIPHER = 'aes-256-cbc'
37
38
 
38
39
  class << self
39
40
  def error(msg)
@@ -222,7 +223,7 @@ module GoodData
222
223
  # encrypts data with the given key. returns a binary data with the
223
224
  # unhashed random iv in the first 16 bytes
224
225
  def encrypt(data, key)
225
- cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
226
+ cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
226
227
  cipher.encrypt
227
228
  cipher.key = key = Digest::SHA256.digest(key)
228
229
  random_iv = cipher.random_iv
@@ -236,7 +237,7 @@ module GoodData
236
237
 
237
238
  # Simple encrypt data with given key
238
239
  def simple_encrypt(data, key)
239
- cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
240
+ cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
240
241
  cipher.encrypt
241
242
  cipher.key = key
242
243
  encrypted = cipher.update(data)
@@ -253,7 +254,7 @@ module GoodData
253
254
 
254
255
  data = Base64.decode64(database64)
255
256
 
256
- cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
257
+ cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
257
258
  cipher.decrypt
258
259
  cipher.key = cipher_key = Digest::SHA256.digest(key)
259
260
  random_iv = data[0..15] # extract iv from first 16 bytes
@@ -273,7 +274,7 @@ module GoodData
273
274
 
274
275
  data = Base64.decode64(database64)
275
276
 
276
- cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
277
+ cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
277
278
  cipher.decrypt
278
279
  cipher.key = key
279
280
  decrypted = cipher.update(data)