gooddata 1.3.6 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e709fa1e08dc151286a4516d9aefb012b3b6a9c7
-  data.tar.gz: a4417002c9f847063d6ee13e872365785e4bda48
+SHA256:
+  metadata.gz: ebb2d24e75eda57c8dca39237e1f0a81a8d9a73324278927fff78b99beb0a1db
+  data.tar.gz: fb22a2ced11bf90f0ff4356ccc361698e20ab80bd8be02161458deaf49d8bade
 SHA512:
-  metadata.gz: 4eef8de1fe6944f76c87a105142450d9ace0e032c4ddab5a04c01f8336669d9041dc53685201e0d67714ecbd6d1889f7060741f8e68184196d596ff3fbc34ce9
-  data.tar.gz: 96d3c2d9b3dd682581bea81c00371f2166df3e694d4b1d7691f923ab557225eafc0fa59b7ce860c709a1de6acd0bacd27a074fff5a83391ce1d894f3c1d20fcf
+  metadata.gz: 3afb795645aedca94fd96cd58cb92d3e5232e898a98cb231d9f7f5dffcad711a68a6eeb313b1ba04a6c72f2d2b296b3132477e703a859c3995036b47c6de219a
+  data.tar.gz: 784b70f590256dfabe834d78ec5e9fe6979d41e27a53449826fc330d2a732f2cf5a287d94d7be83a23a72920238b826caa87d26a1fe36493ca35bd3303af0d26

data/.gitignore

@@ -13,6 +13,7 @@ tmtags
 ## VIM
 *.swp
 .tags*
+tags*
 
 ## IDEA
 .idea/

data/.travis.yml

@@ -7,6 +7,8 @@ stages:
   if: type IN (api, cron)
 - name: platform-cleanup
   if: type IN (cron)
+- name: gem-release
+  if: branch = master AND type = push
 
 git:
   depth: false # this is needed for pronto
@@ -71,10 +73,11 @@ jobs:
   - &lcm-slow-tests
     name: staging1 - lcm slow tests
     stage: after-merge
-    script: |
-      bundle exec rake -f lcm.rake docker:build
-      bundle exec rake -f lcm.rake docker:bundle
-      bundle exec rake -f lcm.rake test:docker:slow
+    script:
+      - sudo keytool -importcert -alias gooddata-2008 -file "./data/2008.crt" -keystore $JAVA_HOME/jre/lib/security/cacerts -trustcacerts -storepass 'changeit' -noprompt
+      - sudo keytool -importcert -alias gooddata-int -file "./data/new_ca.cer" -keystore $JAVA_HOME/jre/lib/security/cacerts -trustcacerts -storepass 'changeit' -noprompt
+      - sudo keytool -importcert -alias gooddata-prod -file "data/new_prodgdc_ca.crt" -keystore $JAVA_HOME/jre/lib/security/cacerts -trustcacerts -storepass 'changeit' -noprompt
+      - bundle exec rake -f lcm.rake test:slow
     env:
     - VCR_ON=false
     - GD_ENV=staging
@@ -187,6 +190,18 @@ jobs:
     name: staging3 - test environment clean-up
     env: GD_ENV=development
 
+  - stage: gem-release
+    rvm:
+      - 2.3
+      - jruby-9.1.14
+    script: echo "Deploying to rubygems.org ..."
+    deploy:
+      provider: rubygems
+      gem: gooddata
+      api_key: $RUBYGEMS_API_KEY
+      on:
+        branch: master
+
 notifications:
   email:
     recipients:

data/CHANGELOG.md

@@ -1,4 +1,40 @@
 # GoodData Ruby SDK Changelog
+## 2.0.0
+
+This is a major version release because the method `.release_info` has been hard-deprecated.
+
+ - Release gem using travis
+ - TMA-1054: Rename monitoring rules definition
+ - TMA-1054: alertingRules for prometheus
+ - TMA-1295: Fix passing file_name to after block
+ - TMA-1295: Don't run slow tests in docker
+ - Fix logging connection stats
+ - Fix spec file pattern
+ - Fix splunk logger test
+ - TMA-1284: Fix passing s3 info to config
+ - Fail when deprecated release_info called
+ - Fix perf cluster appstore name
+ - FEATURE: GDP-539 remove of releaseInfo resource
+ - Recover schedule helper file
+ - Fix load tests
+ - Record vcr cassettes
+ - Run load tests on the server
+ - switched user groups cache from instance variable to param
+ - TMA-1060: correct the smoke test to allow testing both dev and stable versions
+ -  TMA-483 lowering the abusement of user groups resource
+ - TMA-807: up to code standards
+ - TMA-807: log profiling data to splunk
+ - Use latest version of Pronto
+ - TMA-1255: subscription spec utilizes project cache
+ - BUGFIX: TMA-1230 Log brick execution errors
+ - BUGFIX: TMA-1230 Log error message when client disconnecting fails
+ - BUGFIX: TMA-1230 Return inner logger class in MaskLoggerDecorator
+ - Fix path to version file in docs
+ - Fix path to version file in version:bump
+ - Mention rake task version:changelog in docs
+ - fix: remove containers after run
+ - SETI-2444: add missing curl-dev package to docker image
+
 ## 1.3.6
  - Make path to version file relative
 
@@ -270,7 +306,7 @@
 
 ## 1.1.0
  - TMA-860: use new roles API (#1169)
- - test passed locally 
+ - test passed locally
  - Avoid polling idle time in VCR tests
  - Enable vcr for metric specs
  - TMA-832: support for pluggable component process type

data/CONTRIBUTING.md

@@ -1,5 +1,9 @@
 # Contributing
 
+## Pull requests
+
+Make pull requests to the `develop` branch. Once all tests are passing on `develop`, we will merge to `master` and release the gem.
+
 ## Tests
 ### CI setup
 gooddata-ruby has a robust CI setup in place to ensure easy contributing by both GD employees and outsiders. The CI is based on travis-ci and all of the environment specifics is versioned in .travis.yml

data/Gemfile

@@ -1,4 +1,13 @@
 source 'https://rubygems.org'
 
+group 'development' do
+  unless RUBY_PLATFORM == 'java'
+    # git because https://github.com/prontolabs/pronto/issues/312
+    gem 'pronto',
+        git: 'https://github.com/prontolabs/pronto',
+        ref: '266805b'
+  end
+end
+
 # Specify your gem's dependencies in gooddata.gemspec
 gemspec

data/Jenkinsfile-chart

@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) 2007-2019, GoodData(R) Corporation. All rights reserved.
+ */
+
+@Library('pipelines-shared-libs')
+import com.gooddata.pipeline.Pipeline
+
+def config = [
+        'charts': [
+                'lcm-bricks'
+        ]
+]
+
+Pipeline.get(this, config).run()

data/RELEASING.md

@@ -3,13 +3,10 @@
 1. `git clone https://github.com/gooddata/gooddata-ruby.git gooddata-ruby`
 1. `cd gooddata-ruby`
 1. `git checkout master`
-1. `rvm use ruby`
-1. `bundle install`
-1. bump version in [lib/gooddata/version.rb](lib/gooddata/version.rb)
+1. optionally check what's changed since last release: `bundle exec rake version:changelog`
+1. bump version in [SDK_VERSION](SDK_VERSION)
 1. `bundle exec rake version:bump`
-1. create PR to upstream and have it merged
+1. create PR to `upstream/develop` and have it merged
 1. `git push origin tags/{version}`
-1. `rake gem:release`
-1. `rvm use jruby && rm Gemfile.lock && bundle install`
-1. `rake gem:release`
-1. release [cookbook](https://github.com/gooddata/gooddata-ruby-examples)
+1. once all tests are passing, merge `develop` to `master` and the gem will be released automatically
+1. release [cookbook](https://github.com/gooddata/gooddata-ruby-doc)

data/Rakefile

@@ -179,7 +179,7 @@ namespace :version do
     require_relative 'lib/gooddata/version'
     new_version = GoodData::VERSION
     update_changelog(new_version)
-    `git add CHANGELOG.md lib/gooddata/version.rb`
+    `git add CHANGELOG.md SDK_VERSION`
     `git commit -m "Bump version to #{new_version}"`
     `git tag #{new_version}`
   end

data/SDK_VERSION

@@ -1 +1 @@
-1.3.6
+2.0.0

data/bin/run_brick.rb

@@ -36,7 +36,7 @@ begin
 rescue NoMethodError => e
   log.info "action=#{brick_type}_execution status=error Invalid brick type '#{brick_type}', #{e.message}"
   raise e
-rescue => e
+rescue StandardError => e
   log.info "action=#{brick_type}_execution status=error #{e.message}"
   raise e
 end

data/bin/test_projects_cleanup.rb

@@ -1,10 +1,21 @@
-example = 'GD_ENV=testing GD_SPEC_PASSWORD=secret bundle exec ruby bin/test_projects_cleanup.rb'
+require 'optparse'
 
-if ARGV.include?('-h')
-  puts 'Example:'
-  puts example
-  exit 0
-end
+example = 'Usage example: GD_ENV=testing GD_SPEC_PASSWORD=secret bundle exec ruby bin/test_projects_cleanup.rb'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = example
+  opts.on('-f', '--force', 'Mercilessly deletes matching projects.') do |v|
+    options[:force] = v
+  end
+  opts.on('-d N', '--days N', Integer, 'Number of days to keep projects for.') do |v|
+    options[:days] = v
+  end
+  opts.on_tail("-h", "--help", "Show this message.") do
+    puts opts
+    exit 0
+  end
+end.parse!
 
 require 'gooddata'
 require_relative '../spec/environment/environment'
@@ -72,6 +83,7 @@ def clean_up!(client, force, days)
   delete_project_by_title(/userprov-e2e-testing/, projects, days, force)
   delete_ads_by_title(/Development ADS/, client, days, force)
   delete_ads_by_title(/Production ADS/, client, days, force)
+  delete_ads_by_title(/^users brick load test/, client, days, force)
 end
 
 def init_client(username, password, server)
@@ -89,8 +101,8 @@ password = secrets[:password]
 dev_client = init_client(username, password, "https://#{config[:dev_server]}")
 prod_client = init_client(username, password, "https://#{config[:prod_server]}")
 
-force = ARGV.include?('-f')
-days = 14
+force = options[:force]
+days = options[:days] || 14
 clean_up!(dev_client, force, days)
 clean_up!(prod_client, force, days)
 

data/ci.rake

@@ -7,11 +7,11 @@ namespace :docker do
   desc 'Bundles gems using cache'
   namespace :bundle do
     task :ruby do
-      system('docker-compose run gooddata-ruby bundle')
+      system('docker-compose run --rm gooddata-ruby bundle')
     end
 
     task :jruby do
-      system('docker-compose run gooddata-jruby bundle')
+      system('docker-compose run --rm gooddata-jruby bundle')
     end
   end
 end

data/k8s/charts/lcm-bricks/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+name: lcm-bricks
+description: LCM Bricks
+version: 1.0.2

data/k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml

@@ -0,0 +1,96 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: lcm-bricks-monitoring-rules
+  namespace: monitoring
+  labels:
+    app: lcm-bricks
+    component: lcm
+    team: lcm
+    chart: lcm-bricks
+    release: {{ .Release.Name }}
+data:
+  lcm-bricks-monitoring-rules.yaml: |+
+    groups:
+    - name: lcm-bricks-monitoring-rules
+      rules:
+      - record: "container_pod:lcm_pod_container_status_restarts:increase10m"
+        expr: increase(kube_pod_container_status_restarts_total{namespace='{{ .Release.Namespace }}'}[10m])
+      - alert: "[LCM] Pod has too many restarts on cluster={{ .Values.clusterId }}"
+        expr: container_pod:lcm_pod_container_status_restarts:increase10m >= 1
+        labels:
+          severity: warning
+          team: lcm # switch to msf in production
+          cluster_id: {{ .Values.clusterId }}
+        annotations:
+          description: "There is more than 0 restarts of {{`{{ $labels.pod }}`}} pod in the last 10 minutes"
+          summary: "{{`{{ $labels.pod }}`}} pod has too many restarts"
+      - alert: "[LCM] Pod has too many restarts on cluster={{ .Values.clusterId }}"
+        expr: container_pod:lcm_pod_container_status_restarts:increase10m >= 2
+        labels:
+          severity: critical
+          team: lcm # switch to msf in production
+          cluster_id: {{ .Values.clusterId }}
+        annotations:
+          description: "There is more than 1 restart of {{`{{ $labels.pod }}`}} pod in the last 10 minutes"
+          summary: "{{`{{ $labels.pod }}`}} pod has too many restarts"
+      - record: "container_pod:lcm_pod_container_status_oomkilled:increase10m"
+        expr: increase(kube_pod_container_status_terminated_reason{namespace='{{ .Release.Namespace }}', reason='OOMKilled'}[10m])
+      - alert: "[LCM] OOMKill occured on cluster={{ .Values.clusterId }}"
+        expr: container_pod:lcm_pod_container_status_oomkilled:increase10m >= 1
+        labels:
+          severity: warning
+          team: lcm # switch to msf in production
+          cluster_id: {{ .Values.clusterId }}
+        annotations:
+          description: "{{`{{ $labels.pod }}`}} was OOMKilled in the last 30 minutes. Investigate and/or increase memoryRequest or memoryLimit."
+          summary: "{{`{{ $labels.pod }}`}} OOMKill occured"
+      - alert: "[LCM] OOMKill occured on cluster={{ .Values.clusterId }}"
+        expr: container_pod:lcm_pod_container_status_oomkilled:increase10m >= 2
+        labels:
+          severity: critical
+          team: lcm # switch to msf in production
+          cluster_id: {{ .Values.clusterId }}
+        annotations:
+          description: "{{`{{ $labels.pod }}`}} was OOMKilled in the last 10 minutes. Investigate and/or increase memoryRequest or memoryLimit."
+          summary: "{{`{{ $labels.pod }}`}} OOMKill occured"
+      - record: "container_pod:lcm_pod_container_status_error:increase10m"
+        expr: increase(kube_pod_container_status_terminated_reason{namespace='{{ .Release.Namespace }}', reason!~"Completed|OOMKilled"}[10m])
+      - alert: "[LCM] Container exited with error on cluster={{ .Values.clusterId }}"
+        expr: container_pod:lcm_pod_container_status_error:increase10m >= 1
+        labels:
+          severity: warning
+          team: lcm # switch to msf in production
+          cluster_id: {{ .Values.clusterId }}
+        annotations:
+          description: "{{`{{ $labels.pod }}`}} container exited with non-zero exit code in the last 10 minutes. Investigate and fix."
+          summary: "{{`{{ $labels.pod }}`}} Container exited with error"
+      - alert: "[LCM] Container exited with error on cluster={{ .Values.clusterId }}"
+        expr: container_pod:lcm_pod_container_status_error:increase10m >= 2
+        labels:
+          severity: critical
+          team: lcm # switch to msf in production
+          cluster_id: {{ .Values.clusterId }}
+        annotations:
+          description: "{{`{{ $labels.pod }}`}} container exited with non-zero exit code in the last 10 minutes. Investigate and fix."
+          summary: "{{`{{ $labels.pod }}`}} Container exited with error"
+      - alert: "[LCM] Container is being throttled on cluster={{ .Values.clusterId }}"
+        expr: rate(container_cpu_cfs_throttled_seconds_total{namespace='{{ .Release.Namespace }}'}[1m]) > 1
+        for: 5m
+        labels:
+          severity: critical
+          team: lcm # switch to msf in production
+          cluster_id: {{ .Values.clusterId }}
+        annotations:
+          description: "{{`{{ $labels.pod_name }}`}} container is beeing throttled and probably hit CPU limit. Investigate root cause and increase limit and/or number of replicas if necessary."
+          summary: "{{`{{ $labels.pod_name }}`}} Container is being throttled"
+      - alert: "[LCM] is doing too much pause GC on cluster={{ .Values.clusterId }}"
+        expr: rate(jvm_gc_pause_seconds_sum{kubernetes_namespace='{{ .Release.Namespace }}'}[1m]) > 1
+        for: 5m
+        labels:
+          severity: critical
+          team: lcm # switch to msf in production
+          cluster_id: {{ .Values.clusterId }}
+        annotations:
+          description: "{{`{{ $labels.kubernetes_pod_name }}`}} container is spending too much time in pause garbage collector. Investigate root cause and increase heap size and/or number of replicas if necessary."
+          summary: "{{`{{ $labels.kubernetes_pod_name }}`}} is doing too much pause GC"

data/lcm.rake

@@ -79,7 +79,7 @@ namespace :test do
   test_cases.each do |test_case|
     desc "Run #{test_case} tests"
     RSpec::Core::RakeTask.new(test_case) do |task|
-      task.pattern = "spec/lcm/#{test_case}/**/*.rb"
+      task.pattern = "spec/lcm/#{test_case}/**/*_spec.rb"
     end
   end
 

data/lib/gooddata/bricks/middleware/context_logger_decorator.rb

@@ -0,0 +1,31 @@
+# Copyright (c) 2010-2018 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  # Logger decorator with ability to enrich the message with brick context
+  module ContextLoggerDecorator
+    attr_accessor :context_source
+
+    # log methods to be decorated
+    def add(severity, message = nil, progname = nil)
+      super(severity, enrich(message, context_source.context), progname)
+    end
+
+    private
+
+    # Enrich given message.
+    # @param message [String] or [Hash] message to enrich
+    # @param context [Hash] context by which the message should be enriched
+    # @return masked_message [String] or [Hash] enriched message
+    def enrich(message, context)
+      if !message
+        context
+      elsif message.is_a?(Hash)
+        context.merge(message)
+      else
+        context.merge(message: message)
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/context_manager.rb

@@ -0,0 +1,68 @@
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative '../../mixins/property_accessor'
+require_relative '../../lcm/helpers/helpers'
+
+module GoodData
+  module ContextManager
+    extend GoodData::Mixin::PropertyAccessor
+
+    property_accessor :@context, :action
+    property_accessor :@context, :brick
+    property_accessor :@context, :execution_id
+    property_accessor :@context, :status
+
+    def initialize_context
+      @action_start = Time.now
+
+      # :log_v is used to differentiate new versions of logs in splunk
+      @context = {
+        :api_version => GoodData.version,
+        :log_v => 0,
+        :component => 'lcm.ruby',
+        :status => :not_in_action
+      }
+    end
+
+    # Return current brick context extended with time specific information
+    #
+    # @param [Time] now, allows to specify exact time, when outer call was performed
+    # @return [Hash] Brick context
+    def context(now = Time.now)
+      time_specific_context = action ? { :time => time_from_action_start(now) } : {}
+      @context.merge(time_specific_context)
+    end
+
+    def time_from_action_start(now = Time.now)
+      fail_if_development 'No action is being profiled' unless action
+      (now - @action_start) * 1000
+    end
+
+    # Starts lcm action
+    #
+    # @param [String] action, name of the action
+    # @param [Logger] logger, logger that should log current context info
+    # @param [Time] now, allows to specify exact time, when outer call was performed
+    def start_action(next_action, logger = nil, now = Time.now)
+      fail_if_development 'An action is already being profiled' if action
+
+      self.action = next_action
+      @action_start = now
+      logger.info '' if logger
+      self.status = :action_in_progress
+    end
+
+    # Ends currently opened lcm action
+    #
+    # @param [Logger] logger, logger that should log current context info
+    def end_action(logger = nil)
+      fail_if_development 'No matching action to start found' unless action
+
+      logger.info '' if logger
+      self.status = :not_in_action
+      self.action = nil
+    end
+  end
+end