gooddata 2.1.8-java → 2.1.13-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +7 -0
  3. data/.travis.yml +2 -4
  4. data/CHANGELOG.md +43 -0
  5. data/Dockerfile +19 -4
  6. data/Dockerfile.jruby +4 -4
  7. data/Dockerfile.ruby +5 -4
  8. data/README.md +2 -0
  9. data/SDK_VERSION +1 -1
  10. data/VERSION +1 -1
  11. data/bin/provision.sh +2 -0
  12. data/bin/release.sh +2 -0
  13. data/bin/rollout.sh +2 -0
  14. data/bin/run_brick.rb +31 -7
  15. data/bin/test_projects_cleanup.rb +10 -2
  16. data/bin/user_filters.sh +2 -0
  17. data/ci.rake +1 -1
  18. data/ci/bigquery/pom.xml +54 -0
  19. data/ci/redshift/pom.xml +73 -0
  20. data/ci/snowflake/pom.xml +57 -0
  21. data/dev-gooddata-sso.pub.encrypted +40 -40
  22. data/gdc_fossa_lcm.yaml +2 -0
  23. data/gdc_fossa_ruby_sdk.yaml +4 -0
  24. data/gooddata.gemspec +6 -2
  25. data/k8s/charts/lcm-bricks/Chart.yaml +1 -1
  26. data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml +22 -12
  27. data/lcm.rake +14 -0
  28. data/lib/gooddata/bricks/middleware/execution_result_middleware.rb +68 -0
  29. data/lib/gooddata/bricks/middleware/logger_middleware.rb +2 -1
  30. data/lib/gooddata/bricks/middleware/mask_logger_decorator.rb +5 -1
  31. data/lib/gooddata/bricks/pipeline.rb +7 -0
  32. data/lib/gooddata/cloud_resources/bigquery/bigquery_client.rb +86 -0
  33. data/lib/gooddata/cloud_resources/bigquery/drivers/.gitkeepme +0 -0
  34. data/lib/gooddata/cloud_resources/cloud_resouce_factory.rb +28 -0
  35. data/lib/gooddata/cloud_resources/cloud_resource_client.rb +24 -0
  36. data/lib/gooddata/cloud_resources/cloud_resources.rb +12 -0
  37. data/lib/gooddata/cloud_resources/redshift/drivers/log4j.properties +15 -0
  38. data/lib/gooddata/cloud_resources/redshift/redshift_client.rb +101 -0
  39. data/lib/gooddata/cloud_resources/snowflake/drivers/.gitkeepme +0 -0
  40. data/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +84 -0
  41. data/lib/gooddata/exceptions/invalid_env_error.rb +15 -0
  42. data/lib/gooddata/helpers/data_helper.rb +10 -0
  43. data/lib/gooddata/helpers/data_source_helpers.rb +47 -0
  44. data/lib/gooddata/helpers/global_helpers.rb +4 -0
  45. data/lib/gooddata/helpers/global_helpers_params.rb +6 -9
  46. data/lib/gooddata/lcm/actions/collect_clients.rb +6 -6
  47. data/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb +6 -6
  48. data/lib/gooddata/lcm/actions/collect_segment_clients.rb +4 -1
  49. data/lib/gooddata/lcm/actions/collect_segments.rb +1 -2
  50. data/lib/gooddata/lcm/actions/collect_users_brick_users.rb +7 -6
  51. data/lib/gooddata/lcm/actions/create_segment_masters.rb +5 -3
  52. data/lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb +116 -0
  53. data/lib/gooddata/lcm/actions/set_master_project.rb +76 -0
  54. data/lib/gooddata/lcm/actions/synchronize_clients.rb +1 -1
  55. data/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +1 -2
  56. data/lib/gooddata/lcm/actions/synchronize_ldm.rb +20 -3
  57. data/lib/gooddata/lcm/actions/synchronize_user_filters.rb +23 -3
  58. data/lib/gooddata/lcm/actions/synchronize_users.rb +50 -30
  59. data/lib/gooddata/lcm/actions/update_release_table.rb +7 -1
  60. data/lib/gooddata/lcm/exceptions/lcm_execution_error.rb +16 -0
  61. data/lib/gooddata/lcm/helpers/release_table_helper.rb +16 -8
  62. data/lib/gooddata/lcm/lcm2.rb +28 -5
  63. data/lib/gooddata/models/domain.rb +17 -15
  64. data/lib/gooddata/models/execution.rb +0 -1
  65. data/lib/gooddata/models/execution_detail.rb +0 -1
  66. data/lib/gooddata/models/from_wire.rb +1 -0
  67. data/lib/gooddata/models/process.rb +11 -3
  68. data/lib/gooddata/models/profile.rb +33 -11
  69. data/lib/gooddata/models/project.rb +120 -31
  70. data/lib/gooddata/models/project_creator.rb +2 -0
  71. data/lib/gooddata/models/schedule.rb +0 -1
  72. data/lib/gooddata/rest/client.rb +2 -2
  73. data/lib/gooddata/rest/connection.rb +5 -3
  74. data/rubydev_public.gpg.encrypted +51 -51
  75. data/rubydev_secret_keys.gpg.encrypted +109 -109
  76. metadata +32 -13
  77. data/lib/gooddata/extensions/hash.rb +0 -18
@@ -0,0 +1,57 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project xmlns="http://maven.apache.org/POM/4.0.0"
3
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5
+ <modelVersion>4.0.0</modelVersion>
6
+
7
+ <groupId>com.gooddata.lcm</groupId>
8
+ <artifactId>lcm-snowflake-driver</artifactId>
9
+ <version>1.0-SNAPSHOT</version>
10
+
11
+ <dependencies>
12
+ <dependency>
13
+ <groupId>net.snowflake</groupId>
14
+ <artifactId>snowflake-jdbc</artifactId>
15
+ <version>3.6.22</version>
16
+ </dependency>
17
+ <dependency>
18
+ <groupId>org.slf4j</groupId>
19
+ <artifactId>slf4j-api</artifactId>
20
+ <version>1.7.2</version>
21
+ </dependency>
22
+ </dependencies>
23
+
24
+ <profiles>
25
+ <profile>
26
+ <id>binary-packaging</id>
27
+ <build>
28
+ <plugins>
29
+ <plugin>
30
+ <artifactId>maven-dependency-plugin</artifactId>
31
+ <executions>
32
+ <execution>
33
+ <phase>package</phase>
34
+ <goals>
35
+ <goal>copy-dependencies</goal>
36
+ </goals>
37
+ <configuration>
38
+ <outputDirectory>${project.build.directory}</outputDirectory>
39
+ <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
40
+ <includeScope>runtime</includeScope>
41
+ </configuration>
42
+ </execution>
43
+ </executions>
44
+ </plugin>
45
+ </plugins>
46
+ </build>
47
+ </profile>
48
+ </profiles>
49
+
50
+ <repositories>
51
+ <repository>
52
+ <id>my-repo1</id>
53
+ <name>my custom repo</name>
54
+ <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
55
+ </repository>
56
+ </repositories>
57
+ </project>
@@ -1,40 +1,40 @@
1
- BDaDY16tjuekrE/7oY03jMefPWuS8GDZjFUvFi0NTtShnvYfk2hQ1Dj2jJxX
2
- grIIadq2m3aGfbpYx8gtgmiCt4I1H4uHhYihz7/hlFcl9FkqdWWS5NIhEula
3
- 5oANQcPQgUoPppdOq/g136Tg+yxhCqjcpRUpooNaFMlt4UHnK0J3VCifYo4o
4
- q0UGZ5VHtiJSAE27eSVi6CI04CtOR/2XBwZFytEyKBabQD89YEvtZgwXvRsO
5
- +A+HZc17HfYgKFhVT333/Xk7wck2cCTpdp0rTrS6Vvh/1a76v6WOqY4UH4T6
6
- NRjyeFjWWXlSAdEts9oqbce0hWohMzFxhruxnYgKYMGUwAsYEONeaJ+b2XP0
7
- gD0LMk+w3nuLldwIEy5x/ciIwhSr6xU1VQ5OR/3YLzDNZKjxEm4paOuQ9ocn
8
- IqyBZEEpASgupQJz77OOyF1lderX6/vC7MwNkjUuMmtRr0q3WwtxildcHKr0
9
- Q4dwCHkq+cLJ7RjtVQlVZUgnFvuZikGSRwnQB0RBSCRMA1ekmuQ3cukk8V7p
10
- LTmWCDW/cJzJcp678r7jOsE+zEIcnZ+LX3OA3eWyvtnZZMby6f/PGjqWbr0E
11
- xlIWj3JXgWVTIXLyaRxYn7JBAy3JqkNKhR/HiggSahlM1+EJff1svJqaIH/N
12
- nfqJP+fo4YW8xcVRd3KNI711T3EZQg8kl0jEZkJn4JPM2Vs7pRukCATuXF3J
13
- C3/4gtSQ9xYHLyPc2l1Ny0AKrfTztWQUqJn4mjxQxIu+Rw9xLw6Tv0887mL1
14
- dHWQ/bVw6vQ0fkARRP1xKAmpJwSZCQEisFDWVRy3ua87EDkvHOT+PHOnpGOB
15
- 8bVflDd8fXJIpfNhQCXQEospQsEOmbSxijy9NxDk9P7QuhvMuXJkLnx4dRXI
16
- RyvhrNUB76FdHlhUQf2vuPyPChMwTabO6sqL6EZKpL4omqrXSogEjBDri1RS
17
- ua+ixPtXY4rC07v+UURt8jCRdcZBqD/H+cTAvcMdDm/339CxMViXZWb2Om6j
18
- 14hWxQRPPlmZXqMJWGHCwr9tsylCZvgeaaCVcRRCRKxccARSBHNuh62eq6ZC
19
- 9X5fSk8Yjbpw1bMKJXTB8moXAmuHgPO0KwHW6V2acJGWSr5yRiXVZ9f38y+g
20
- Fwvg/S4Fs8ESOh08Sh16ExIbHl98+DP6qyqRHpXKkMlGe2W3n+AxgPu1NBeb
21
- 3TJVw6lF6/YEU1lbgYfqY4WFlT7orN0Jn0ZlDQpQPOF8bFaMrTs6ioP2X5g9
22
- k1T0pN2Zc4/oQl292DoqA8WapoqUJxv+5u1jLcQSsj592qKH0y/bJ0vasj5B
23
- TbPgUWxNnNdZLkAeLY5SKfE8/mI7AHPc2tCJzMPERzW/43Xyi4T22h1SVHtD
24
- ouK+H6BQHsOIvryDew7wRD6e17E7OYRPVTAXFOPFkOhsnKxLzG6gN+IJxvqM
25
- RsqLNc/po0IbySnlY1B5zsKn1QRs6A3Onu6xj4gOSUVDGs1a8L1xNWkqGiOh
26
- OQ9b6AaKJZG2qqImBHHOYEeZ1QB9clLJ3EawU6PiKxwwZOVZlU5qI1SasZIB
27
- WtRyeiSQQD1dKUVEkISI7+oVfS4OI07tlMRpzuJtrvYCV0ASImYyqD83lZFm
28
- 3ZhSeUaXNhsYp9ac7Q+YRkLEKKbbWqtHG7LSkXVNpTbDVgzNIJiaj17bo6r0
29
- ZZLcW1Kqna9DxY/PLE5JbQS/CmRaPRt2VLKyKyqoSYQFdbdRzJyAIe0PT+6A
30
- 2qEFRLkovlx7kdkqQCbEvnHN/C+cXdyKf/8EtYliGkLRoGfLc6xenQba14U9
31
- 3aslQvItKtsRk0owEvHntag8uWZNmFVyEFDaVVG7peXyn5uigal+hXi5B8DJ
32
- AGOrZhycpHOJ+2HL9RlTrYps0KPKVSNgiauVpW3ZghugtfCF7sxX31+HxQAC
33
- 4p/4kdy2R7R0cjqG8OjZ/xb3isSxhGSSjShh3RgCEJyRg55Zs+gR/XvEKk97
34
- kCiz8k8IbSmVdWLUb17Txy5hmGZJYakohdZuO8+h0vYl2SVIH8jkgsRF6Wad
35
- TtA+EQshJ87qqAugQjdR29Dk9NCL2VVFuebl647F0gnh538Kxm2i745W963f
36
- hblQuo+JPWUi/b4nMJXRBu8+/8/wnZfBl2gb2rvVFaSVF44ndJFPr06eFUFm
37
- eygWl/IlTNArWPHJJG4BRdIvzrvLCM8+3NcSQAWCsWp1ZaiTfw5NbKrP1Eze
38
- 3bpKOdy5IKDdh5ACdXeX3l0CXGY53otgCpb9s87VrCDXFtA+wQU6kshNf1M6
39
- EuI/Fqx8XnPSLOdEH9TyAKVN0FEMfSVOqk9l8LQf1vqNh0jcjMcy18LKDTkz
40
- B+9dpPw=
1
+ 08UExe8tEw64I9RbLcz/HtM/Jynhv84Fos5f2z6sPS/RJxCsx1CULoG7SfCy
2
+ udzARn2teIjlgW8H+F5g5OTmK5pyJXRnQBPUm+Dzf0lkWqY3ZLKNhlJLV/2d
3
+ dFQV3TyOTF+wMk5ENhZG12txrgEi832tX8o2eLuNx5r0NRmW0zbzFDJKlwPC
4
+ kcY/eDWIsb+KAQxJeR3GpMFQ/Wm5Wo9OaaR8QIJ5rHR+YHdCPy3Xfu0vndYN
5
+ w/Mie1yuOnMeq9ieWwGQBhxcGMqJJSKt0jsl6+Ecs/8GJ8BULhTHrEUTiqBa
6
+ 3mrl2AiIIXPVuTtVVZOfJkNnct5pfOO7oEEhpafhJ9OsJROipvc3/gYLohRI
7
+ xHytF0pOtABd2WjIchttpSqy9n6NCuzh2ULdfDziedJm270jqs1l1siYQV6q
8
+ bs4ER0kDXJoUSPVN+kmbTca8K0tfjSYYA2LPbhxYG5oYriWL/uAp4BztnEkW
9
+ Cscc6UJ3Z6CL99JfTp+kKwvE7AHDyWjJsUKNclRmXO33i4vkJf7wAy+E9/8w
10
+ IUm2lWLpQybeIYwGJj6Ku72c9ZPBZyA9gfVPLDo5yR4YTZtUT6WQbn7FjbXl
11
+ jTjg5DBUQuJYs75WxV62XRZAJjvYfk+nhv4CvMXdh2nntvATr2hRLZ2RNbTG
12
+ lPOzX0oDBiSdWD4vKQZq2/dc8vrGdCT27s7konQRbUpxEh/5lBhc92LNQsgB
13
+ V7agBWVfr+ASB6g1PQ480pXEI3BRmjmR/+5jIJ4VN9p4qyGqYFS5Q3wZV86t
14
+ +UNNOJhDHgr+H7hm4rLXpMYp2Fki7D/Xaijaq11t65ARCqcFpwtSj68NNeMv
15
+ Y1Tq1ypcOjUGAszYc6K7Ngso4ZWZ8JAiR7lHZP5kHTvd/wG6f/EbavoEjpoW
16
+ NHuYZ7JPZLCmRVRjPa1ivd7h0CfkwJpKn2bUHusgXNgmKRWoAUaQ3piGRNqa
17
+ zyeNJJsZSSbfiC7U1lbj+uMBNPCPTtyAo0ad3IaVhZjzeK614MZN+kebDqLO
18
+ 4OmN2JYc5wWNz8PwKTLEH1Chwk9k0l2d2o3C6JSO0EKu9XrIfxXfAGSpTTwt
19
+ h1krSicVjbYGzrYExm9Uha9xHjGdiJiv+PEPCEfKTbZI1HzqRbMbT9sRNnWK
20
+ RmPpN3BGvYFs1g2lz7yNrCNZC8FamUCQPBL1h2xLohqKaWbNXbYq5Nild6NK
21
+ ZFjp2UukfHm2+fgyDzMZNeEg40t6dLD5L7N+hx8FL0z+7hLVpksCDlom+U8P
22
+ 3EiClDFt6f6RLApUl3j+1lj4rc4jvUVqZ+ckmxDBKJEU2E9nM7ixx5oqybWf
23
+ YuNI1AnZCpvDPpzWhJHlxlRu/YIzwJ8fyCPgPGbwZu+pGemd9TvWmBZevZvn
24
+ GYZuxYuwPQhDORUA2FchF+BmLRu4hQtKJv8AEOCdikdu/F4WWM0dvfMRWUSo
25
+ 57nomhI3UUMm0AI9zxplLXUXIyt4cIL3owgb186C3VHLQRgedQ2ns5iRGRlS
26
+ BEnWK4zoi4ERS/4JGi08sL4c4vydsIHYcsD0M7cM2qURfHSFESGbNGQnaIiE
27
+ +CTXvkqIIqwxmZ+Ff26AC0cUtoeymxxe5U54j5RVcv8ij9VvP4rrH0lkuJB/
28
+ Ulz7hD38A9A3PwqqNRPnoXVFke/bBL9N0QXKNFOJpZ6ltZMXjZ7CEq2k0Bq5
29
+ +yM8PU7inoNEPjWaS9lPrfqTjTzrGFF62vgc6raRdde6LRTxfggA3pNxxOvK
30
+ 5WRedMkrwBSAFGbnsSb12qYYuZiaERLX5GtcHyw4AOmK5N49JLcSuvnX77YZ
31
+ iNP04r4XV2oq1j8pAp6KZQU/j0q17HXWFBopGyGjE18cPpo4bR3NxChodFKe
32
+ QMwgs8gX3xSdZd4UaYaUNKBZvmyxfb2gFTykfSetqikMpbTYxIzPz9ITz8E+
33
+ CoJA3vwnEuBJTI9Owl1lX9FoqTU2wwNPALqhghf5cXseK/UB4evzGzC1gxOx
34
+ nTpG+JkKyqRoiT5XmOidU+CZUA95IYhi5mGEMIqwmsS8HYR8qyULsvtN9RoW
35
+ yI9aTA48UfTN/VazBpTomxrA44ChjLMR7WFwdWXaNGw00faCdhm0fLZo2Kox
36
+ NTBZMhZQarV9RAUU5aozujhkuU+YZVD8CG40RoHFjeeZ8RhODVlntepUC1p1
37
+ ALonjlg+GgzuFtodh6qe/Qx1zjNGLHaT1T/02hWl4ERYSMxkoGPLC7dPjLdc
38
+ liQtPQFLFlEDX6mwmOe7Mcnz90ThyaO7K3MhnMjOFx4cTF84RDaD4Omj+U5Z
39
+ N7k9HVGhYz5VSGvI/dO2j0QFrxwEPt1VVRPT0hlzKsrxSPI2jMtVFhLF2AH/
40
+ PHRjAjc=
@@ -0,0 +1,2 @@
1
+ ---
2
+ fossa_project: "gooddata-ruby-lcm"
@@ -0,0 +1,4 @@
1
+ ---
2
+ fossa_project: "gooddata-ruby-sdk"
3
+ ignored_paths:
4
+ - 'ci/.*'
@@ -49,12 +49,16 @@ Gem::Specification.new do |s|
49
49
  s.add_development_dependency 'pronto', '~> 0.10' if RUBY_PLATFORM != 'java'
50
50
  s.add_development_dependency 'pronto-rubocop', '~> 0.9' if RUBY_PLATFORM != 'java'
51
51
  s.add_development_dependency 'pronto-reek', '~> 0.9' if RUBY_PLATFORM != 'java'
52
- s.add_development_dependency 'vcr'
52
+ s.add_development_dependency 'vcr', '5.0.0'
53
53
  s.add_development_dependency 'hashdiff', '~> 0.4'
54
54
 
55
55
  s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
56
56
 
57
- s.add_dependency 'activesupport', '> 4.2.9', '< 6.1'
57
+ if RUBY_VERSION >= '2.5'
58
+ s.add_dependency 'activesupport', '>= 6.0.3.1', '< 6.1'
59
+ else
60
+ s.add_dependency 'activesupport', '>= 5.2.4.3', '< 6.0'
61
+ end
58
62
 
59
63
  s.add_dependency 'aws-sdk-s3', '~> 1.16'
60
64
  s.add_dependency 'docile', '~> 1.1'
@@ -1,4 +1,4 @@
1
1
  apiVersion: v1
2
2
  name: lcm-bricks
3
3
  description: LCM Bricks
4
- version: 2.0.1
4
+ version: 2.0.3
@@ -20,7 +20,7 @@ data:
20
20
  expr: container_pod:lcm_pod_container_status_restarts:increase10m >= 1
21
21
  labels:
22
22
  severity: warning
23
- team: lcm # switch to msf in production
23
+ team: lcm
24
24
  cluster_id: {{ .Values.clusterId }}
25
25
  annotations:
26
26
  description: "There is more than 0 restarts of {{`{{ $labels.pod }}`}} pod in the last 10 minutes"
@@ -28,8 +28,8 @@ data:
28
28
  - alert: "[LCM] Pod has too many restarts on cluster={{ .Values.clusterId }}"
29
29
  expr: container_pod:lcm_pod_container_status_restarts:increase10m >= 2
30
30
  labels:
31
- severity: critical
32
- team: lcm # switch to msf in production
31
+ severity: warning
32
+ team: lcm
33
33
  cluster_id: {{ .Values.clusterId }}
34
34
  annotations:
35
35
  description: "There is more than 1 restart of {{`{{ $labels.pod }}`}} pod in the last 10 minutes"
@@ -40,7 +40,7 @@ data:
40
40
  expr: container_pod:lcm_pod_container_status_oomkilled:increase10m >= 1
41
41
  labels:
42
42
  severity: warning
43
- team: lcm # switch to msf in production
43
+ team: lcm
44
44
  cluster_id: {{ .Values.clusterId }}
45
45
  annotations:
46
46
  description: "{{`{{ $labels.pod }}`}} was OOMKilled in the last 30 minutes. Investigate and/or increase memoryRequest or memoryLimit."
@@ -48,8 +48,8 @@ data:
48
48
  - alert: "[LCM] OOMKill occured on cluster={{ .Values.clusterId }}"
49
49
  expr: container_pod:lcm_pod_container_status_oomkilled:increase10m >= 2
50
50
  labels:
51
- severity: critical
52
- team: lcm # switch to msf in production
51
+ severity: warning
52
+ team: lcm
53
53
  cluster_id: {{ .Values.clusterId }}
54
54
  annotations:
55
55
  description: "{{`{{ $labels.pod }}`}} was OOMKilled in the last 10 minutes. Investigate and/or increase memoryRequest or memoryLimit."
@@ -58,8 +58,8 @@ data:
58
58
  expr: rate(container_cpu_cfs_throttled_seconds_total{namespace='{{ .Release.Namespace }}'}[1m]) > 1
59
59
  for: 5m
60
60
  labels:
61
- severity: critical
62
- team: lcm # switch to msf in production
61
+ severity: warning
62
+ team: lcm
63
63
  cluster_id: {{ .Values.clusterId }}
64
64
  annotations:
65
65
  description: "{{`{{ $labels.pod_name }}`}} container is beeing throttled and probably hit CPU limit. Investigate root cause and increase limit and/or number of replicas if necessary."
@@ -68,8 +68,8 @@ data:
68
68
  expr: rate(jvm_gc_pause_seconds_sum{kubernetes_namespace='{{ .Release.Namespace }}'}[1m]) > 1
69
69
  for: 5m
70
70
  labels:
71
- severity: critical
72
- team: lcm # switch to msf in production
71
+ severity: warning
72
+ team: lcm
73
73
  cluster_id: {{ .Values.clusterId }}
74
74
  annotations:
75
75
  description: "{{`{{ $labels.kubernetes_pod_name }}`}} container is spending too much time in pause garbage collector. Investigate root cause and increase heap size and/or number of replicas if necessary."
@@ -77,9 +77,19 @@ data:
77
77
  - alert: "[LCM] there is more than 100 jobs on cluster={{ .Values.clusterId }}"
78
78
  expr: count(kube_job_info{namespace="lcm"}) > 100
79
79
  labels:
80
- severity: critical
81
- team: lcm # switch to msf in production
80
+ severity: warning
81
+ team: lcm
82
82
  cluster_id: {{ .Values.clusterId }}
83
83
  annotations:
84
84
  description: "There is more than 100 jobs in LCM namespace. They are likely not deleted."
85
85
  summary: "There is more than 100 jobs in LCM namespace."
86
+ - alert: "[LCM] Resource quotas hit CPU limit on cluster={{ .Values.clusterId }}"
87
+ expr: kube_resourcequota{namespace='{{ .Release.Namespace }}',resource="limits.cpu",type="hard"} - ignoring(type) kube_resourcequota{namespace='{{ .Release.Namespace }}',resource="limits.cpu",type="used"} == 0
88
+ labels:
89
+ severity: warning
90
+ team: lcm
91
+ cluster_id: {{ .Values.clusterId }}
92
+ annotations:
93
+ description: "We are hitting CPU limit in LCM namespace."
94
+ summary: "We are hitting CPU limit in LCM namespace."
95
+
data/lcm.rake CHANGED
@@ -118,6 +118,7 @@ end
118
118
  namespace :docker do
119
119
  desc 'Build Docker image'
120
120
  task :build do
121
+ Rake::Task["maven:build_dependencies"].invoke
121
122
  system('docker build -f Dockerfile.jruby -t gooddata/appstore .')
122
123
  end
123
124
 
@@ -127,6 +128,19 @@ namespace :docker do
127
128
  end
128
129
  end
129
130
 
131
+ namespace :maven do
132
+ task :build_dependencies do
133
+ system('mvn -f ci/snowflake/pom.xml clean install -P binary-packaging')
134
+ system('cp -rf ci/snowflake/target/*.jar lib/gooddata/cloud_resources/snowflake/drivers/')
135
+
136
+ system('mvn -f ci/bigquery/pom.xml clean install -P binary-packaging')
137
+ system('cp -rf ci/bigquery/target/*.jar lib/gooddata/cloud_resources/bigquery/drivers/')
138
+
139
+ system('mvn -f ci/redshift/pom.xml clean install -P binary-packaging')
140
+ system('cp -rf ci/redshift/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
141
+ end
142
+ end
143
+
130
144
  namespace :sdk do
131
145
  desc 'Updates gooddata-ruby to the version specified in the root Gemfile'
132
146
  task :update do
@@ -0,0 +1,68 @@
1
+ # Copyright (c) 2019, GoodData Corporation. All rights reserved.
2
+ # This source code is licensed under the BSD-style license found in the
3
+ # LICENSE file in the root directory of this source tree.
4
+
5
+ require_relative 'base_middleware'
6
+
7
+ # rubocop:disable Style/ClassVars
8
+ module GoodData
9
+ module Bricks
10
+ module ExecutionStatus
11
+ OK = 'OK'
12
+ ERROR = 'ERROR'
13
+ WARNING = 'WARNING'
14
+ end
15
+
16
+ class ExecutionResultMiddleware < Bricks::Middleware
17
+ @@result_log_path = nil
18
+
19
+ def call(params)
20
+ result_log_path(params)
21
+ @app.call(params)
22
+ end
23
+
24
+ # Update process execution result when the script is executed inside a GDC ETL process execution (ruby bricks).
25
+ # Ruby bricks should update execution result at the end of script with status=WARNING or OK and summary message if the script exits normally.
26
+ # If any fatal error, script should update execution result with status=ERROR and error message,
27
+ # then throw exception to notify GDC platform that the script terminated unexpectedly.
28
+ # @param [HashMap] params contains GDC_EXECUTION_RESULT_LOG_PATH or GDC_LOG_DIRECTORY, GDC_EXECUTION_ID
29
+ # @param [ExecutionStatus] status execution status
30
+ # @param [String] message execution message
31
+ def self.update_execution_result(status, message = "")
32
+ if status != ExecutionStatus::OK && status != ExecutionStatus::ERROR && status != ExecutionStatus::WARNING
33
+ GoodData.logger.warn("Unknown execution status #{status}, ignored it.")
34
+ end
35
+
36
+ result = {
37
+ executionResult: {
38
+ status: status,
39
+ message: message
40
+ }
41
+ }
42
+ update_result(result)
43
+ end
44
+
45
+ private
46
+
47
+ def result_log_path(params)
48
+ log_directory = params['GDC_LOG_DIRECTORY']
49
+ execution_id = params['GDC_EXECUTION_ID']
50
+ result_log_path = params['GDC_EXECUTION_RESULT_LOG_PATH'] || ENV['GDC_EXECUTION_RESULT_LOG_PATH']
51
+ result_log_path = "#{log_directory}/#{execution_id}_result.json" if result_log_path.nil? && !log_directory.nil?
52
+ @@result_log_path = result_log_path
53
+ end
54
+
55
+ def self.update_result(result)
56
+ if @@result_log_path.nil?
57
+ GoodData.gd_logger.warn("action=update_execution_result status=error Not found execution result logger file.") unless GoodData.gd_logger.nil?
58
+ return
59
+ end
60
+
61
+ File.open(@@result_log_path, 'w') { |file| file.write(JSON.pretty_generate(result)) }
62
+ rescue Exception => e # rubocop:disable RescueException
63
+ GoodData.gd_logger.error("action=update_execution_result status=error reason=#{e.message}") unless GoodData.gd_logger.nil?
64
+ end
65
+ end
66
+ end
67
+ end
68
+ # rubocop:enable Style/ClassVars
@@ -36,7 +36,8 @@ module GoodData
36
36
  log_directory = params['GDC_LOG_DIRECTORY']
37
37
  execution_id = params['GDC_EXECUTION_ID']
38
38
  FileUtils.mkpath log_directory
39
- logger = Logger.new("#{log_directory}/#{execution_id}.log")
39
+ execution_log_path = params['GDC_EXECUTION_LOG_PATH'].nil? ? "#{log_directory}/#{execution_id}.log" : params['GDC_EXECUTION_LOG_PATH']
40
+ logger = Logger.new(execution_log_path)
40
41
  logger.level = params['GDC_LOG_LEVEL'] || 'info'
41
42
  values_to_mask = params['values_to_mask'] || []
42
43
  logger = MaskLoggerDecorator.new(logger, values_to_mask)
@@ -20,7 +20,11 @@ module GoodData
20
20
  # @return [[String]] array of all String in values
21
21
  def extract_values(values)
22
22
  if values.is_a?(String)
23
- [values]
23
+ if values.nil? || values.empty? || values.gsub(/[*|\s]/, '') == ''
24
+ []
25
+ else
26
+ [values]
27
+ end
24
28
  elsif values.is_a?(Hash) || values.is_a?(Array)
25
29
  (values.is_a?(Hash) ? values.values : values).reduce([]) do |strings, item|
26
30
  strings.concat extract_values(item)
@@ -32,6 +32,7 @@ module GoodData
32
32
  prepare([
33
33
  LoggerMiddleware,
34
34
  DecodeParamsMiddleware,
35
+ ExecutionResultMiddleware,
35
36
  BenchMiddleware,
36
37
  GoodDataMiddleware,
37
38
  AWSMiddleware,
@@ -44,6 +45,7 @@ module GoodData
44
45
  prepare([
45
46
  LoggerMiddleware,
46
47
  DecodeParamsMiddleware,
48
+ ExecutionResultMiddleware,
47
49
  BenchMiddleware,
48
50
  GoodDataMiddleware,
49
51
  AWSMiddleware,
@@ -56,6 +58,7 @@ module GoodData
56
58
  def self.release_brick_pipeline
57
59
  prepare([
58
60
  LoggerMiddleware,
61
+ ExecutionResultMiddleware,
59
62
  DecodeParamsMiddleware,
60
63
  BenchMiddleware,
61
64
  GoodDataMiddleware,
@@ -68,6 +71,7 @@ module GoodData
68
71
  def self.provisioning_brick_pipeline
69
72
  prepare([
70
73
  LoggerMiddleware,
74
+ ExecutionResultMiddleware,
71
75
  DecodeParamsMiddleware,
72
76
  BenchMiddleware,
73
77
  GoodDataMiddleware,
@@ -80,6 +84,7 @@ module GoodData
80
84
  def self.rollout_brick_pipeline
81
85
  prepare([
82
86
  LoggerMiddleware,
87
+ ExecutionResultMiddleware,
83
88
  DecodeParamsMiddleware,
84
89
  BenchMiddleware,
85
90
  GoodDataMiddleware,
@@ -93,6 +98,7 @@ module GoodData
93
98
  prepare(
94
99
  [
95
100
  LoggerMiddleware,
101
+ ExecutionResultMiddleware,
96
102
  DecodeParamsMiddleware,
97
103
  BenchMiddleware,
98
104
  HelloWorldBrick
@@ -104,6 +110,7 @@ module GoodData
104
110
  prepare(
105
111
  [
106
112
  LoggerMiddleware,
113
+ ExecutionResultMiddleware,
107
114
  DecodeParamsMiddleware,
108
115
  BenchMiddleware,
109
116
  HelpBrick
@@ -0,0 +1,86 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+ #
4
+ # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
5
+ # This source code is licensed under the BSD-style license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ require 'securerandom'
9
+ require 'java'
10
+ require 'pathname'
11
+ require_relative '../cloud_resource_client'
12
+
13
+ base = Pathname(__FILE__).dirname.expand_path
14
+ Dir.glob(base + 'drivers/*.jar').each do |file|
15
+ require file unless file.start_with?('lcm-bigquery-driver')
16
+ end
17
+
18
+ java_import 'com.google.auth.oauth2.ServiceAccountCredentials'
19
+ java_import 'com.google.cloud.bigquery.BigQuery'
20
+ java_import 'com.google.cloud.bigquery.BigQueryOptions'
21
+ java_import 'com.google.cloud.bigquery.FieldList'
22
+ java_import 'com.google.cloud.bigquery.FieldValue'
23
+ java_import 'com.google.cloud.bigquery.FieldValueList'
24
+ java_import 'com.google.cloud.bigquery.QueryJobConfiguration'
25
+ java_import 'com.google.cloud.bigquery.TableResult'
26
+ java_import 'org.apache.commons.text.StringEscapeUtils'
27
+
28
+ module GoodData
29
+ module CloudResources
30
+ class BigQueryClient < CloudResourceClient
31
+ class << self
32
+ def accept?(type)
33
+ type == 'bigquery'
34
+ end
35
+ end
36
+
37
+ def initialize(options = {})
38
+ raise("Data Source needs a client to BigQuery to be able to query the storage but 'bigquery_client' is empty.") unless options['bigquery_client']
39
+
40
+ if options['bigquery_client']['connection'].is_a?(Hash)
41
+ @project = options['bigquery_client']['connection']['project']
42
+ @schema = options['bigquery_client']['connection']['schema'] || 'public'
43
+ @authentication = options['bigquery_client']['connection']['authentication']
44
+ else
45
+ raise('Missing connection info for BigQuery client')
46
+
47
+ end
48
+ end
49
+
50
+ def realize_query(query, _params)
51
+ GoodData.gd_logger.info("Realize SQL query: type=bigquery status=started")
52
+
53
+ client = create_client
54
+ filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
55
+ measure = Benchmark.measure do
56
+ query_config = QueryJobConfiguration.newBuilder(query).setDefaultDataset(@schema).build
57
+ table_result = client.query(query_config)
58
+
59
+ if table_result.getTotalRows.positive?
60
+ result = table_result.iterateAll
61
+ field_list = table_result.getSchema.getFields
62
+ col_count = field_list.size
63
+ CSV.open(filename, 'wb') do |csv|
64
+ csv << Array(1..col_count).map { |i| field_list.get(i - 1).getName } # build the header
65
+ result.each do |row|
66
+ csv << Array(1..col_count).map { |i| row.get(i - 1).getValue&.to_s }
67
+ end
68
+ end
69
+ end
70
+ end
71
+ GoodData.gd_logger.info("Realize SQL query: type=bigquery status=finished duration=#{measure.real}")
72
+ filename
73
+ end
74
+
75
+ private
76
+
77
+ def create_client
78
+ GoodData.logger.info "Setting up connection to BigQuery"
79
+ client_email = @authentication['serviceAccount']['clientEmail']
80
+ private_key = @authentication['serviceAccount']['privateKey']
81
+ credentials = ServiceAccountCredentials.fromPkcs8(nil, client_email, StringEscapeUtils.unescapeJson(private_key), nil, nil)
82
+ BigQueryOptions.newBuilder.setProjectId(@project).setCredentials(credentials).build.getService
83
+ end
84
+ end
85
+ end
86
+ end