gooddata 2.1.14-java → 2.2.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +5 -5
  2. data/.gdc-ii-config.yaml +1 -1
  3. data/.github/workflows/build.yml +66 -0
  4. data/.github/workflows/pre-merge.yml +72 -0
  5. data/.sonar.settings +4 -0
  6. data/.travis.yml +78 -12
  7. data/CHANGELOG.md +62 -0
  8. data/Dockerfile +25 -14
  9. data/LICENSE +4418 -17
  10. data/LICENSE.rb +1 -1
  11. data/README.md +3 -3
  12. data/Rakefile +8 -1
  13. data/SDK_VERSION +1 -1
  14. data/VERSION +1 -1
  15. data/bin/test_projects_cleanup.rb +45 -3
  16. data/ci/mssql/pom.xml +62 -0
  17. data/ci/mysql/pom.xml +57 -0
  18. data/ci/postgresql/pom.xml +57 -0
  19. data/ci/redshift/pom.xml +1 -1
  20. data/dev-gooddata-sso.pub.encrypted +40 -40
  21. data/gdc_fossa_ruby_sdk.yaml +1 -0
  22. data/gooddata.gemspec +10 -6
  23. data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
  24. data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml +11 -1
  25. data/lcm.rake +2 -2
  26. data/lib/gooddata/bricks/middleware/aws_middleware.rb +35 -9
  27. data/lib/gooddata/cloud_resources/blobstorage/blobstorage_client.rb +98 -0
  28. data/lib/gooddata/cloud_resources/{cloud_resouce_factory.rb → cloud_resource_factory.rb} +8 -0
  29. data/lib/gooddata/cloud_resources/cloud_resources.rb +1 -1
  30. data/lib/gooddata/cloud_resources/mssql/drivers/.gitkeepme +0 -0
  31. data/lib/gooddata/cloud_resources/mssql/mssql_client.rb +122 -0
  32. data/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme +0 -0
  33. data/lib/gooddata/cloud_resources/mysql/mysql_client.rb +111 -0
  34. data/lib/gooddata/cloud_resources/postgresql/drivers/.gitkeepme +0 -0
  35. data/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +106 -0
  36. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +18 -1
  37. data/lib/gooddata/commands/scaffold.rb +9 -10
  38. data/lib/gooddata/core/nil_logger.rb +3 -1
  39. data/lib/gooddata/helpers/data_helper.rb +9 -5
  40. data/lib/gooddata/helpers/global_helpers.rb +6 -5
  41. data/lib/gooddata/lcm/actions/associate_clients.rb +8 -2
  42. data/lib/gooddata/lcm/actions/base_action.rb +0 -2
  43. data/lib/gooddata/lcm/actions/collect_meta.rb +3 -1
  44. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +3 -2
  45. data/lib/gooddata/lcm/actions/provision_clients.rb +31 -10
  46. data/lib/gooddata/lcm/actions/synchronize_clients.rb +56 -7
  47. data/lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb +64 -0
  48. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +19 -8
  49. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +16 -9
  50. data/lib/gooddata/lcm/actions/synchronize_users.rb +7 -6
  51. data/lib/gooddata/lcm/actions/update_metric_formats.rb +185 -0
  52. data/lib/gooddata/lcm/data/delete_from_lcm_release.sql.erb +5 -0
  53. data/lib/gooddata/lcm/helpers/release_table_helper.rb +42 -8
  54. data/lib/gooddata/lcm/lcm2.rb +5 -2
  55. data/lib/gooddata/lcm/types/base_type.rb +0 -2
  56. data/lib/gooddata/mixins/md_object_query.rb +9 -6
  57. data/lib/gooddata/models/blueprint/project_blueprint.rb +0 -2
  58. data/lib/gooddata/models/client.rb +14 -12
  59. data/lib/gooddata/models/data_source.rb +668 -0
  60. data/lib/gooddata/models/dataset_mapping.rb +36 -0
  61. data/lib/gooddata/models/domain.rb +3 -2
  62. data/lib/gooddata/models/metadata/analytical_dashboard.rb +49 -0
  63. data/lib/gooddata/models/metadata/analytical_visualization_object.rb +30 -0
  64. data/lib/gooddata/models/metadata/label.rb +26 -27
  65. data/lib/gooddata/models/metadata/visualization_object.rb +50 -0
  66. data/lib/gooddata/models/project.rb +66 -19
  67. data/lib/gooddata/models/schedule.rb +13 -1
  68. data/lib/gooddata/models/user_filters/user_filter_builder.rb +58 -54
  69. data/lib/gooddata/models/user_group.rb +0 -1
  70. data/lib/gooddata/rest/connection.rb +6 -4
  71. data/lib/gooddata/rest/phmap.rb +2 -1
  72. data/lib/gooddata.rb +2 -0
  73. data/rubydev_public.gpg.encrypted +51 -51
  74. data/rubydev_secret_keys.gpg.encrypted +109 -109
  75. metadata +51 -27
  76. data/DEPENDENCIES.md +0 -880
  77. data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb +0 -37
  78. data/lib/gooddata/helpers/data_source_helpers.rb +0 -47
@@ -0,0 +1,98 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'pathname'
10
+ require "azure/storage/blob"
11
+
12
+ module GoodData
13
+ class BlobStorageClient
14
+ SAS_URL_PATTERN = %r{(^https?:\/\/[^\/]*)\/.*\?(.*)}
15
+ INVALID_BLOB_GENERAL_MESSAGE = "The connection string is not valid."
16
+ INVALID_BLOB_SIG_WELL_FORMED_MESSAGE = "The signature format is not valid."
17
+ INVALID_BLOB_CONTAINER_MESSAGE = "ContainerNotFound"
18
+ INVALID_BLOB_CONTAINER_FORMED_MESSAGE = "The container with the specified name is not found."
19
+ INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE = "Signature not valid in the specified time frame"
20
+ INVALID_BLOB_EXPIRED_MESSAGE = "The signature expired."
21
+ INVALID_BLOB_INVALID_CONNECTION_STRING_MESSAGE = "The connection string is not valid."
22
+ INVALID_BLOB_PATH_MESSAGE = "BlobNotFound"
23
+ INVALID_BLOB_INVALID_PATH_MESSAGE = "The path to the data is not found."
24
+
25
+ attr_reader :use_sas
26
+
27
+ def initialize(options = {})
28
+ raise("Data Source needs a client to Blob Storage to be able to get blob file but 'blobStorage_client' is empty.") unless options['blobStorage_client']
29
+
30
+ if options['blobStorage_client']['connectionString'] && options['blobStorage_client']['container']
31
+ @connection_string = options['blobStorage_client']['connectionString']
32
+ @container = options['blobStorage_client']['container']
33
+ @path = options['blobStorage_client']['path']
34
+ @use_sas = false
35
+ build_sas(@connection_string)
36
+ else
37
+ raise('Missing connection info for Blob Storage client')
38
+ end
39
+ end
40
+
41
+ def realize_blob(file, _params)
42
+ GoodData.gd_logger.info("Realizing download from Blob Storage. Container #{@container}.")
43
+ filename = ''
44
+ begin
45
+ connect
46
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
47
+ blob_name = get_blob_name(@path, file)
48
+
49
+ measure = Benchmark.measure do
50
+ _blob, content = @client.get_blob(@container, blob_name)
51
+ File.open(filename, "wb") { |f| f.write(content) }
52
+ end
53
+ rescue => e
54
+ raise_error(e)
55
+ end
56
+ GoodData.gd_logger.info("Done downloading file type=blobStorage status=finished duration=#{measure.real}")
57
+ filename
58
+ end
59
+
60
+ def connect
61
+ GoodData.logger.info "Setting up connection to Blob Storage"
62
+ if use_sas
63
+ @client = Azure::Storage::Blob::BlobService.create(:storage_blob_host => @host, :storage_sas_token => @sas_token)
64
+ else
65
+ @client = Azure::Storage::Blob::BlobService.create_from_connection_string(@connection_string)
66
+ end
67
+ end
68
+
69
+ def build_sas(url)
70
+ matches = url.scan(SAS_URL_PATTERN)
71
+ return unless matches && matches[0]
72
+
73
+ @use_sas = true
74
+ @host = matches[0][0]
75
+ @sas_token = matches[0][1]
76
+ end
77
+
78
+ def raise_error(e)
79
+ if e.message && e.message.include?(INVALID_BLOB_EXPIRED_ORIGINAL_MESSAGE)
80
+ raise INVALID_BLOB_EXPIRED_MESSAGE
81
+ elsif e.message && e.message.include?(INVALID_BLOB_SIG_WELL_FORMED_MESSAGE)
82
+ raise INVALID_BLOB_SIG_WELL_FORMED_MESSAGE
83
+ elsif e.message && e.message.include?(INVALID_BLOB_CONTAINER_MESSAGE)
84
+ raise INVALID_BLOB_CONTAINER_FORMED_MESSAGE
85
+ elsif e.message && e.message.include?(INVALID_BLOB_PATH_MESSAGE)
86
+ raise INVALID_BLOB_INVALID_PATH_MESSAGE
87
+ else
88
+ raise INVALID_BLOB_GENERAL_MESSAGE
89
+ end
90
+ end
91
+
92
+ def get_blob_name(path, file)
93
+ return file unless path
94
+
95
+ path.rindex('/') == path.length - 1 ? "#{path}#{file}" : "#{path}/#{file}"
96
+ end
97
+ end
98
+ end
@@ -11,7 +11,15 @@ module GoodData
11
11
  module CloudResources
12
12
  class CloudResourceFactory
13
13
  class << self
14
+ def load_cloud_resource(type)
15
+ base = "#{Pathname(__FILE__).dirname.expand_path}#{File::SEPARATOR}#{type}#{File::SEPARATOR}"
16
+ Dir.glob(base + '**/*.rb').each do |file|
17
+ require file
18
+ end
19
+ end
20
+
14
21
  def create(type, data = {}, opts = {})
22
+ load_cloud_resource(type)
15
23
  clients = CloudResourceClient.descendants.select { |c| c.respond_to?("accept?") && c.send("accept?", type) }
16
24
  raise "DataSource does not support type \"#{type}\"" if clients.empty?
17
25
 
@@ -7,6 +7,6 @@
7
7
  require 'pathname'
8
8
 
9
9
  base = Pathname(__FILE__).dirname.expand_path
10
- Dir.glob(base + '**/*.rb').each do |file|
10
+ Dir.glob(base + '*.rb').each do |file|
11
11
  require file
12
12
  end
@@ -0,0 +1,122 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mssql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MSSQLClient < CloudResourceClient
21
+ MSSQL_SEPARATOR_PARAM = ";"
22
+ MSSQL_URL_PATTERN = %r{jdbc:sqlserver://([^:]+)(:([0-9]+))?(/)?}
23
+ MSSQL_DEFAULT_PORT = 1433
24
+ LOGIN_TIME_OUT = 30
25
+ MSSQL_FETCH_SIZE = 10_000
26
+ VERIFY_FULL = 'verify-full'
27
+ PREFER = 'prefer'
28
+ REQUIRE = 'require'
29
+
30
+ class << self
31
+ def accept?(type)
32
+ type == 'mssql'
33
+ end
34
+ end
35
+
36
+ def initialize(options = {})
37
+ raise("Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") unless options['mssql_client']
38
+
39
+ connection = options['mssql_client']['connection']
40
+ if connection.is_a?(Hash)
41
+ @database = connection['database']
42
+ @schema = connection['schema']
43
+ @authentication = connection['authentication']
44
+ @ssl_mode = connection['sslMode']
45
+ @url = connection['url']
46
+
47
+ validate
48
+ else
49
+ raise('Missing connection info for MSSQL client')
50
+ end
51
+
52
+ Java.com.microsoft.sqlserver.jdbc.SQLServerDriver
53
+ end
54
+
55
+ def realize_query(query, _params)
56
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=started")
57
+
58
+ connect
59
+
60
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
61
+ measure = Benchmark.measure do
62
+ statement = @connection.create_statement
63
+ statement.set_fetch_size(MSSQL_FETCH_SIZE)
64
+ has_result = statement.execute(query)
65
+ if has_result
66
+ result = statement.get_result_set
67
+ metadata = result.get_meta_data
68
+ col_count = metadata.column_count
69
+ CSV.open(filename, 'wb') do |csv|
70
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
71
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
72
+ end
73
+ end
74
+ end
75
+
76
+ GoodData.gd_logger.info("Realize SQL query: type=mssql status=finished duration=#{measure.real}")
77
+ filename
78
+ ensure
79
+ @connection&.close
80
+ @connection = nil
81
+ end
82
+
83
+ def connect
84
+ connection_string = build_connection_string
85
+ GoodData.logger.info "Setting up connection to MSSQL #{connection_string}"
86
+
87
+ authentication = @authentication['basic'] || @authentication['activeDirectoryPassword']
88
+
89
+ prop = java.util.Properties.new
90
+ prop.setProperty('userName', authentication['userName'])
91
+ prop.setProperty('password', authentication['password'])
92
+
93
+ @connection = java.sql.DriverManager.getConnection(connection_string, prop)
94
+ end
95
+
96
+ def validate
97
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
98
+
99
+ raise "Instance name is not supported" if @url !~ /^[^\\]*$/
100
+
101
+ raise "The connection url is invalid. Parameter is not supported." if @url.include? MSSQL_SEPARATOR_PARAM
102
+
103
+ url_matches = @url.scan(MSSQL_URL_PATTERN)
104
+ raise "Cannot reach the url" if url_matches.nil? || url_matches.length.zero?
105
+
106
+ raise "The authentication method is not supported." unless @authentication['basic'] || @authentication['activeDirectoryPassword']
107
+ end
108
+
109
+ def build_connection_string
110
+ encrypt = @ssl_mode != PREFER
111
+ trust_server_certificate = @ssl_mode == REQUIRE
112
+
113
+ "#{@url};" \
114
+ "database=#{@database};" \
115
+ "encrypt=#{encrypt};" \
116
+ "trustServerCertificate=#{trust_server_certificate};" \
117
+ "loginTimeout=#{LOGIN_TIME_OUT};" \
118
+ "#{'authentication=ActiveDirectoryPassword;' if @authentication['activeDirectoryPassword']}"
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,111 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-mysql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class MysqlClient < CloudResourceClient
21
+ JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
22
+ MYSQL_DEFAULT_PORT = 3306
23
+ JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
24
+ VERIFY_FULL = 'VERIFY_IDENTITY'
25
+ PREFER = 'PREFERRED'
26
+ REQUIRE = 'REQUIRED'
27
+ MYSQL_FETCH_SIZE = 1000
28
+
29
+ class << self
30
+ def accept?(type)
31
+ type == 'mysql'
32
+ end
33
+ end
34
+
35
+ def initialize(options = {})
36
+ raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
37
+
38
+ if options['mysql_client']['connection'].is_a?(Hash)
39
+ @database = options['mysql_client']['connection']['database']
40
+ @authentication = options['mysql_client']['connection']['authentication']
41
+ @ssl_mode = options['mysql_client']['connection']['sslMode']
42
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
43
+
44
+ @url = build_url(options['mysql_client']['connection']['url'])
45
+ else
46
+ raise('Missing connection info for Mysql client')
47
+ end
48
+
49
+ Java.com.mysql.cj.jdbc.Driver
50
+ end
51
+
52
+ def realize_query(query, _params)
53
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
54
+
55
+ connect
56
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
57
+ measure = Benchmark.measure do
58
+ statement = @connection.create_statement
59
+ statement.set_fetch_size(MYSQL_FETCH_SIZE)
60
+ has_result = statement.execute(query)
61
+ if has_result
62
+ result = statement.get_result_set
63
+ metadata = result.get_meta_data
64
+ col_count = metadata.column_count
65
+ CSV.open(filename, 'wb') do |csv|
66
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
67
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
68
+ end
69
+ end
70
+ end
71
+ GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
72
+ filename
73
+ ensure
74
+ @connection&.close
75
+ @connection = nil
76
+ end
77
+
78
+ def connect
79
+ GoodData.logger.info "Setting up connection to Mysql #{@url}"
80
+
81
+ prop = java.util.Properties.new
82
+ prop.setProperty('user', @authentication['basic']['userName'])
83
+ prop.setProperty('password', @authentication['basic']['password'])
84
+
85
+ @connection = java.sql.DriverManager.getConnection(@url, prop)
86
+ @connection.set_auto_commit(false)
87
+ end
88
+
89
+ def build_url(url)
90
+ matches = url.scan(JDBC_MYSQL_PATTERN)
91
+ raise 'Cannot reach the url' unless matches
92
+
93
+ host = matches[0][0]
94
+ port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
95
+
96
+ "#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true&enabledTLSProtocols=TLSv1.2"
97
+ end
98
+
99
+ def get_ssl_mode(ssl_mode)
100
+ mode = PREFER
101
+ if ssl_mode == 'verify-full'
102
+ mode = VERIFY_FULL
103
+ elsif ssl_mode == 'require'
104
+ mode = REQUIRE
105
+ end
106
+
107
+ mode
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,106 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2021 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-postgresql-driver')
16
+ end
17
+
18
+ module GoodData
19
+ module CloudResources
20
+ class PostgresClient < CloudResourceClient
21
+ JDBC_POSTGRES_PATTERN = %r{jdbc:postgresql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
22
+ POSTGRES_DEFAULT_PORT = 5432
23
+ JDBC_POSTGRES_PROTOCOL = 'jdbc:postgresql://'
24
+ SSL_JAVA_FACTORY = '&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory'
25
+ VERIFY_FULL = 'verify-full'
26
+ PREFER = 'prefer'
27
+ REQUIRE = 'require'
28
+ POSTGRES_SET_SCHEMA_COMMAND = "set search_path to"
29
+ POSTGRES_FETCH_SIZE = 1000
30
+
31
+ class << self
32
+ def accept?(type)
33
+ type == 'postgresql'
34
+ end
35
+ end
36
+
37
+ def initialize(options = {})
38
+ raise("Data Source needs a client to Postgres to be able to query the storage but 'postgresql_client' is empty.") unless options['postgresql_client']
39
+
40
+ if options['postgresql_client']['connection'].is_a?(Hash)
41
+ @database = options['postgresql_client']['connection']['database']
42
+ @schema = options['postgresql_client']['connection']['schema'] || 'public'
43
+ @authentication = options['postgresql_client']['connection']['authentication']
44
+ @ssl_mode = options['postgresql_client']['connection']['sslMode']
45
+ raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
46
+
47
+ @url = build_url(options['postgresql_client']['connection']['url'])
48
+ else
49
+ raise('Missing connection info for Postgres client')
50
+ end
51
+
52
+ Java.org.postgresql.Driver
53
+ end
54
+
55
+ def realize_query(query, _params)
56
+ GoodData.gd_logger.info("Realize SQL query: type=postgresql status=started")
57
+
58
+ connect
59
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
60
+ measure = Benchmark.measure do
61
+ statement = @connection.create_statement
62
+ statement.set_fetch_size(POSTGRES_FETCH_SIZE)
63
+ has_result = statement.execute(query)
64
+ if has_result
65
+ result = statement.get_result_set
66
+ metadata = result.get_meta_data
67
+ col_count = metadata.column_count
68
+ CSV.open(filename, 'wb') do |csv|
69
+ csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
70
+ csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
71
+ end
72
+ end
73
+ end
74
+ GoodData.gd_logger.info("Realize SQL query: type=postgresql status=finished duration=#{measure.real}")
75
+ filename
76
+ ensure
77
+ @connection&.close
78
+ @connection = nil
79
+ end
80
+
81
+ def connect
82
+ GoodData.logger.info "Setting up connection to Postgresql #{@url}"
83
+
84
+ prop = java.util.Properties.new
85
+ prop.setProperty('user', @authentication['basic']['userName'])
86
+ prop.setProperty('password', @authentication['basic']['password'])
87
+ prop.setProperty('schema', @schema)
88
+
89
+ @connection = java.sql.DriverManager.getConnection(@url, prop)
90
+ statement = @connection.create_statement
91
+ statement.execute("#{POSTGRES_SET_SCHEMA_COMMAND} #{@schema}")
92
+ @connection.set_auto_commit(false)
93
+ end
94
+
95
+ def build_url(url)
96
+ matches = url.scan(JDBC_POSTGRES_PATTERN)
97
+ raise 'Cannot reach the url' unless matches
98
+
99
+ host = matches[0][0]
100
+ port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
101
+
102
+ "#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
103
+ end
104
+ end
105
+ end
106
+ end
@@ -18,6 +18,9 @@ end
18
18
  module GoodData
19
19
  module CloudResources
20
20
  class SnowflakeClient < CloudResourceClient
21
+ SNOWFLAKE_GDC_APPLICATION_PARAMETER = 'application=GoodData_Platform'
22
+ SNOWFLAKE_SEPARATOR_PARAM = '?'
23
+
21
24
  class << self
22
25
  def accept?(type)
23
26
  type == 'snowflake'
@@ -31,7 +34,7 @@ module GoodData
31
34
  @database = options['snowflake_client']['connection']['database']
32
35
  @schema = options['snowflake_client']['connection']['schema'] || 'public'
33
36
  @warehouse = options['snowflake_client']['connection']['warehouse']
34
- @url = options['snowflake_client']['connection']['url']
37
+ @url = build_url(options['snowflake_client']['connection']['url'])
35
38
  @authentication = options['snowflake_client']['connection']['authentication']
36
39
  else
37
40
  raise('Missing connection info for Snowflake client')
@@ -79,6 +82,20 @@ module GoodData
79
82
 
80
83
  @connection = java.sql.DriverManager.getConnection(@url, prop)
81
84
  end
85
+
86
+ def build_url(url)
87
+ is_contain = url.include?(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
88
+ unless is_contain
89
+ if url.include?(SNOWFLAKE_SEPARATOR_PARAM)
90
+ url.concat("&")
91
+ else
92
+ url.concat(SNOWFLAKE_SEPARATOR_PARAM)
93
+ end
94
+ url.concat(SNOWFLAKE_GDC_APPLICATION_PARAMETER)
95
+ end
96
+
97
+ url
98
+ end
82
99
  end
83
100
  end
84
101
  end
@@ -4,7 +4,7 @@
4
4
  # This source code is licensed under the BSD-style license found in the
5
5
  # LICENSE file in the root directory of this source tree.
6
6
 
7
- require 'erubis'
7
+ require 'erb'
8
8
  require 'fileutils'
9
9
  require 'pathname'
10
10
 
@@ -18,15 +18,14 @@ module GoodData
18
18
  # TODO: Add option for custom output dir
19
19
  def project(name)
20
20
  fail ArgumentError, 'No name specified' if name.nil?
21
-
22
21
  FileUtils.mkdir(name)
23
22
  FileUtils.cd(name) do
24
23
  FileUtils.mkdir('model')
25
24
  FileUtils.cd('model') do
26
25
  input = File.read(TEMPLATES_PATH + 'project/model/model.rb.erb')
27
- eruby = Erubis::Eruby.new(input)
26
+ erb = ERB.new(input)
28
27
  File.open('model.rb', 'w') do |f|
29
- f.write(eruby.result(:name => name))
28
+ f.write(erb.result_with_hash(:name => name))
30
29
  end
31
30
  end
32
31
 
@@ -36,9 +35,9 @@ module GoodData
36
35
  end
37
36
 
38
37
  input = File.read(TEMPLATES_PATH + 'project/Goodfile.erb')
39
- eruby = Erubis::Eruby.new(input)
38
+ erb = ERB.new(input)
40
39
  File.open('Goodfile', 'w') do |f|
41
- f.write(eruby.result)
40
+ f.write(erb.result)
42
41
  end
43
42
  end
44
43
  end
@@ -51,15 +50,15 @@ module GoodData
51
50
  FileUtils.mkdir(name)
52
51
  FileUtils.cd(name) do
53
52
  input = File.read(TEMPLATES_PATH + 'bricks/brick.rb.erb')
54
- eruby = Erubis::Eruby.new(input)
53
+ erb = ERB.new(input)
55
54
  File.open('brick.rb', 'w') do |f|
56
- f.write(eruby.result)
55
+ f.write(erb.result)
57
56
  end
58
57
 
59
58
  input = File.read(TEMPLATES_PATH + 'bricks/main.rb.erb')
60
- eruby = Erubis::Eruby.new(input)
59
+ erb = ERB.new(input)
61
60
  File.open('main.rb', 'w') do |f|
62
- f.write(eruby.result)
61
+ f.write(erb.result)
63
62
  end
64
63
  end
65
64
  end
@@ -1,4 +1,5 @@
1
- # Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
1
+ # frozen_string_literal: true
2
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
2
3
  # This source code is licensed under the BSD-style license found in the
3
4
  # LICENSE file in the root directory of this source tree.
4
5
 
@@ -11,6 +12,7 @@ module GoodData
11
12
  @level = nil
12
13
  end
13
14
 
15
+ # No body define need for dummy logger
14
16
  def debug(*_args)
15
17
  end
16
18
 
@@ -44,11 +44,14 @@ module GoodData
44
44
  realize_link
45
45
  when 's3'
46
46
  realize_s3(params)
47
- when 'redshift', 'snowflake', 'bigquery'
47
+ when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
48
48
  raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
49
-
50
49
  require_relative '../cloud_resources/cloud_resources'
51
50
  realize_cloud_resource(source, params)
51
+ when 'blobStorage'
52
+ require_relative '../cloud_resources/blobstorage/blobstorage_client'
53
+ blob_storage_client = GoodData::BlobStorageClient.new(params)
54
+ blob_storage_client.realize_blob(@options[:file], params)
52
55
  else
53
56
  raise "DataSource does not support type \"#{source}\""
54
57
  end
@@ -112,12 +115,13 @@ module GoodData
112
115
  end
113
116
 
114
117
  def realize_s3(params)
115
- s3_client = params['aws_client'] && params['aws_client']['s3_client']
118
+ s3_client = params['s3_client'] && params['s3_client']['client']
116
119
  raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
117
120
  bucket_name = @options[:bucket]
118
- key = @options[:key]
121
+ key = @options[:key].present? ? @options[:key] : @options[:file]
119
122
  raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
120
- raise 'Key "key" is missing in S3 datasource' if key.blank?
123
+ raise 'Key "key" or "file" is missing in S3 datasource' if key.blank?
124
+
121
125
  GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
122
126
  filename = Digest::SHA256.new.hexdigest(@options.to_json)
123
127
  bucket = s3_client.bucket(bucket_name)
@@ -1,6 +1,6 @@
1
1
  # encoding: UTF-8
2
2
  #
3
- # Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
3
+ # Copyright (c) 2010-2021 GoodData Corporation. All rights reserved.
4
4
  # This source code is licensed under the BSD-style license found in the
5
5
  # LICENSE file in the root directory of this source tree.
6
6
 
@@ -34,6 +34,7 @@ module GoodData
34
34
  end
35
35
 
36
36
  set_const :GD_MAX_RETRY, (ENV['GD_MAX_RETRY'] && ENV['GD_MAX_RETRY'].to_i) || 12
37
+ AES_256_CBC_CIPHER = 'aes-256-cbc'
37
38
 
38
39
  class << self
39
40
  def error(msg)
@@ -222,7 +223,7 @@ module GoodData
222
223
  # encrypts data with the given key. returns a binary data with the
223
224
  # unhashed random iv in the first 16 bytes
224
225
  def encrypt(data, key)
225
- cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
226
+ cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
226
227
  cipher.encrypt
227
228
  cipher.key = key = Digest::SHA256.digest(key)
228
229
  random_iv = cipher.random_iv
@@ -236,7 +237,7 @@ module GoodData
236
237
 
237
238
  # Simple encrypt data with given key
238
239
  def simple_encrypt(data, key)
239
- cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
240
+ cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
240
241
  cipher.encrypt
241
242
  cipher.key = key
242
243
  encrypted = cipher.update(data)
@@ -253,7 +254,7 @@ module GoodData
253
254
 
254
255
  data = Base64.decode64(database64)
255
256
 
256
- cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
257
+ cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
257
258
  cipher.decrypt
258
259
  cipher.key = cipher_key = Digest::SHA256.digest(key)
259
260
  random_iv = data[0..15] # extract iv from first 16 bytes
@@ -273,7 +274,7 @@ module GoodData
273
274
 
274
275
  data = Base64.decode64(database64)
275
276
 
276
- cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
277
+ cipher = OpenSSL::Cipher::Cipher.new(AES_256_CBC_CIPHER)
277
278
  cipher.decrypt
278
279
  cipher.key = key
279
280
  decrypted = cipher.update(data)