gooddata 0.6.53 → 0.6.54

data/Dockerfile.jruby

@@ -5,15 +5,13 @@ MAINTAINER Tomas Korcak <korczis@gmail.com>
 RUN apk add --no-cache curl make gcc git g++ python linux-headers binutils-gold gnupg libstdc++
 
 # Switch to directory with sources
-WORKDIR /src
-
-# Copy required stuff
-ADD . .
+WORKDIR /var/lib/jenkins-slave
 
 RUN gem update --system \
-    && gem install bundler \
-    && bundle install
+    && gem install bundler
 
-ENTRYPOINT ["bundle", "exec"]
+ENV BUNDLE_PATH=/bundle
+
+ADD . .
 
 CMD ["./bin/gooddata"]

data/Dockerfile.ruby

@@ -2,18 +2,18 @@ FROM ruby:2.3-alpine
 
 MAINTAINER Tomas Korcak <korczis@gmail.com>
 
-RUN apk add --no-cache curl make gcc git g++ python linux-headers binutils-gold gnupg libstdc++ openssl
+RUN apk add --no-cache curl make gcc git g++ python linux-headers binutils-gold gnupg libstdc++ openssl cmake
 
-# Switch to directory with sources
-WORKDIR /src
+RUN ln -s /usr/bin/make /usr/bin/gmake
 
-# Copy required stuff
-ADD . .
+# Switch to directory with sources
+WORKDIR /var/lib/jenkins-slave
 
 RUN gem update --system \
-    && gem install bundler \
-    && bundle install
+    && gem install bundler
 
-ENTRYPOINT ["bundle", "exec"]
+ENV BUNDLE_PATH=/bundle
+
+ADD . .
 
 CMD ["./bin/gooddata"]

data/Rakefile

@@ -17,6 +17,7 @@ require 'rubocop/rake_task'
 desc 'Run RuboCop'
 RuboCop::RakeTask.new(:cop) do |task|
   task.patterns = ['{lib,spec}/**/*.rb']
+  task.options = ['--force-exclusion']
 end
 
 desc 'Run continuous integration test'
@@ -141,6 +142,29 @@ namespace :license do
   end
 end
 
+namespace :changelog do
+  desc 'Updates the changelog with commit messages'
+  task :update do
+    require_relative 'lib/gooddata/version'
+    new_version = GoodData::VERSION
+    changelog = File.read('CHANGELOG.md')
+    fail 'the version is already mentioned in the changelog' if changelog =~ /## #{new_version}/
+    puts "Creating changelog for version #{new_version}"
+    current_commit = `git rev-parse HEAD`.chomp
+    last_release = %x(git describe --tags `git rev-list --tags --max-count=1`)
+    last_release_commit = `git rev-parse #{last_release}`.chomp
+    changes = `git log --format=%B --no-merges #{last_release_commit}..#{current_commit}`.split("\n").reject(&:empty?)
+    changelog_header = '# GoodData Ruby SDK Changelog'
+    changelog.slice! changelog_header
+    File.open('CHANGELOG.md', 'w+') do |file|
+      file.puts changelog_header + "\n"
+      file.puts "## #{new_version}"
+      changes.each { |change| file.puts ' - ' + change }
+      file.puts changelog
+    end
+  end
+end
+
 RSpec::Core::RakeTask.new(:test)
 
 namespace :test do

data/ci.rake

@@ -0,0 +1,47 @@
+namespace :docker do
+  desc 'Build Docker image'
+  task :build do
+    system('docker-compose build')
+  end
+
+  desc 'Bundles gems using cache'
+  namespace :bundle do
+    task :ruby do
+      system('docker-compose run gooddata-ruby bundle')
+    end
+
+    task :jruby do
+      system('docker-compose run gooddata-jruby bundle')
+    end
+  end
+end
+
+namespace :test do
+  task :test do
+    system('docker-compose run gooddata-ruby bundle exec echo ahoj')
+  end
+end
+
+namespace :pronto do
+  desc 'Performs automated code review on the PR'
+  task :ci do
+    system('docker-compose run gooddata-ruby bundle exec pronto run -f github_pr -c origin/develop --exit-code') ||
+      fail('Pronto execution failed!')
+  end
+end
+
+namespace :test do
+  namespace :unit do
+    task :docker do
+      system('docker-compose run -u 1002:1002 -e HOME=/var/lib/jenkins-slave gooddata-ruby bundle exec rake test:unit') ||
+        fail('Test execution failed!')
+    end
+  end
+
+  namespace :integration do
+    task :docker do
+      system('docker-compose run -u 1002:1002 -e HOME=/var/lib/jenkins-slave gooddata-jruby bundle exec rake test:integration') ||
+        fail('Test execution failed!')
+    end
+  end
+end

data/docker-compose.yml

@@ -0,0 +1,34 @@
+version: '2'
+services:
+  gooddata-ruby:
+    image: gooddata/ruby
+    build:
+      context: .
+      dockerfile: Dockerfile.ruby
+    environment:
+      - PRONTO_GITHUB_ACCESS_TOKEN
+      - PRONTO_PULL_REQUEST_ID
+      - GD_ENV
+    volumes:
+      - /etc/passwd:/etc/passwd:ro
+      - .:/var/lib/jenkins-slave
+    volumes_from:
+      - bundle
+  gooddata-jruby:
+    image: gooddata/jruby
+    build:
+      context: .
+      dockerfile: Dockerfile.jruby
+    environment:
+      - GD_PROJECT_TOKEN
+      - GD_SPEC_PASSWORD
+      - GD_ENV
+    volumes:
+      - /etc/passwd:/etc/passwd:ro
+      - .:/var/lib/jenkins-slave
+    volumes_from:
+      - bundle
+  bundle:
+    image: busybox
+    volumes:
+      - /bundle

data/gooddata.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'redcarpet', '~> 3.1' if RUBY_PLATFORM != 'java'
   s.add_development_dependency 'rspec', '~> 3.5'
   s.add_development_dependency 'rspec-expectations', '~> 3.5'
+  s.add_development_dependency 'rspec_junit_formatter', '~> 0.3.0'
   s.add_development_dependency 'rubocop', '< 0.48' # TODO: remove it after dealing with new rules introduced in 0.48 version
   s.add_development_dependency 'simplecov', '~> 0.12'
   s.add_development_dependency 'webmock', '~> 1.21'
@@ -41,20 +42,25 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'yard-rspec', '~> 0.1'
   s.add_development_dependency 'ZenTest', '~> 4.11'
   s.add_development_dependency 'pry'
+  s.add_development_dependency 'activesupport', '~> 4.2.9'
+  s.add_development_dependency 'pronto', '~> 0.9.5' if RUBY_PLATFORM != 'java'
+  s.add_development_dependency 'pronto-rubocop', '~> 0.9.0' if RUBY_PLATFORM != 'java'
+  s.add_development_dependency 'pronto-reek', '~> 0.9.0' if RUBY_PLATFORM != 'java'
+  s.add_development_dependency 'pronto-flay', '~> 0.9.0' if RUBY_PLATFORM != 'java'
 
   s.add_dependency 'aws-sdk', '~> 2.9', '>= 2.9.42'
   s.add_dependency 'docile', '~> 1.1'
   s.add_dependency 'erubis', '~> 2.7'
   s.add_dependency 'gli', '~> 2.15'
   s.add_dependency 'gooddata_datawarehouse' if RUBY_PLATFORM == 'java'
-  s.add_dependency 'gooddata-dss-jdbc' if RUBY_PLATFORM == 'java'
+  s.add_dependency 'gooddata-dss-jdbc', '0.1.12' if RUBY_PLATFORM == 'java'
   s.add_dependency 'highline', '~> 1.7'
   s.add_dependency 'json_pure', '~> 1.8'
   s.add_dependency 'multi_json', '~> 1.12'
   s.add_dependency 'parseconfig', '~> 1.0'
   s.add_dependency 'pmap', '~> 1.1'
   s.add_dependency 'restforce', '~> 2.4'
-  s.add_dependency 'rest-client', '~> 1.8'
+  s.add_dependency 'rest-client', '~> 2.0'
   s.add_dependency 'rubyzip', '~> 1.2'
   s.add_dependency 'salesforce_bulk_query', '~> 0.2'
   s.add_dependency 'terminal-table', '~> 1.7'

data/lib/gooddata/bricks/middleware/restforce_middleware.rb

@@ -4,9 +4,6 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-# TODO: Remove this
-# require 'gooddata'
-
 require_relative 'base_middleware'
 
 module GoodData

data/lib/gooddata/helpers/data_helper.rb

@@ -97,7 +97,7 @@ module GoodData
             open(link) { |rf| f.write(rf.read) }
           end
         end
-        puts "Realizing web download from \"#{link}\" took #{measure.real}"
+        GoodData.logger.info("Realizing web download from \"#{link}\" took #{measure.real}")
         filename
       end
 
@@ -109,16 +109,19 @@ module GoodData
         key = @options[:key]
         raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
         raise 'Key "key" is missing in S3 datasource' if key.blank?
-        puts "Realizing download from S3. Bucket #{bucket_name}, object with key #{key}."
+        GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
         filename = Digest::SHA256.new.hexdigest(@options.to_json)
         bucket = s3_client.bucket(bucket_name)
         obj = bucket.object(key)
-        File.open(filename, 'wb') do |file|
-          obj.get do |chunk|
-            file.write(chunk)
-          end
+        obj.get(response_target: filename, bucket: bucket_name, key: key)
+        s3_size = obj.size
+        actual_size = File.size(filename)
+        GoodData.logger.info("File size in S3: #{s3_size}")
+        GoodData.logger.info("Downloaded file size: #{actual_size}")
+        unless s3_size == actual_size
+          fail "Error downloading file #{key}. Expected size #{s3_size}, got #{actual_size}."
         end
-        puts 'Done downloading file.'
+        GoodData.logger.info('Done downloading file.')
         filename
       end
     end

data/lib/gooddata/helpers/global_helpers_params.rb

@@ -264,9 +264,14 @@ module GoodData
               elsif match =~ /\\\$/
                 '$'
               elsif match =~ /\$\{(\w+)\}/
-                val = params["#{$1}"] || raise("The gd_encoded_params parameter contains unknow reference #{$1}") # rubocop: disable Style/PerlBackrefs
-                reference_values << val
-                val
+                val = params["#{$1}"]
+                if val
+                  reference_values << val
+                  val
+                else
+                  GoodData.logger.warn "Reference '#{$1}' is not found!"
+                  match
+                end
               end
             end
           end

data/lib/gooddata/lcm/actions/apply_custom_maql.rb

@@ -44,9 +44,10 @@ module GoodData
 
           domain_name = params.organization || params.domain
           domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
+          data_product = params.data_product
 
           segment_ids = params.segments.map(&:segment_id)
-          domain_segments = domain.segments.select do |ds|
+          domain_segments = domain.segments(:all, data_product).select do |ds|
             segment_ids.include?(ds.segment_id)
           end
 

data/lib/gooddata/lcm/actions/associate_clients.rb

@@ -45,14 +45,23 @@ module GoodData
 
           domain_name = params.organization || params.domain
           domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
+          data_product = params.data_product
+
+          segments = {}
 
           params.clients.group_by { |data| data[:segment] }.each do |segment_name, clients|
-            segment = domain.segments(segment_name)
+            segment = domain.segments(segment_name, data_product)
+            segments[segment_name] = segment
             (clients.map(&:id) - segment.clients.map(&:id)).each do |c|
               segment.create_client(id: c)
             end
           end
 
+          params.clients.map! do |data|
+            data[:data_product_id] = segments[data[:segment]].data_product.data_product_id
+            data
+          end
+
           domain.update_clients_settings(params.clients)
 
           delete_projects = GoodData::Helpers.to_boolean(params.delete_projects)

data/lib/gooddata/lcm/actions/collect_client_projects.rb

@@ -0,0 +1,78 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+require 'thread_safe'
+
+module GoodData
+  module LCM2
+    class CollectClientProjects < BaseAction
+      DESCRIPTION = 'Collect All Client Projects In Domain'
+
+      PARAMS = define_params(self) do
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+
+        description 'Organization Name'
+        param :organization, instance_of(Type::StringType), required: true
+
+        description 'Segments to manage'
+        param :segments, array_of(instance_of(Type::SegmentType)), required: true
+      end
+
+      RESULT_HEADER = [
+        :client_id,
+        :project
+      ]
+
+      class << self
+        def call(params)
+          client = params.gdc_gd_client
+
+          domain = client.domain(params.organization) || fail("Invalid domain name specified - #{params.organization}")
+
+          data_product = params.data_product
+          all_segments = domain.segments(:all, data_product)
+
+          segment_names = params.segments.map do |segment|
+            segment.segment_id.downcase
+          end
+
+          segments = all_segments.select do |segment|
+            segment_names.include?(segment.segment_id.downcase)
+          end
+
+          client_projects = ThreadSafe::Hash.new
+
+          results = segments.pmap do |segment|
+            segment.clients.map do |segment_client|
+              project = segment_client.project
+
+              res = {
+                client_id: segment_client.client_id,
+                project: project && project.pid
+              }
+
+              client_projects[segment_client.client_id] = {
+                segment_client: segment_client,
+                project: project
+              }
+
+              res
+            end
+          end
+
+          {
+            results: results.flatten,
+            params: {
+              client_projects: client_projects
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/collect_clients.rb

@@ -47,7 +47,8 @@ module GoodData
         def call(params)
           segment_names = params.segments.map(&:segment_id)
 
-          clients = collect_clients(params, segment_names)
+          clients, errors = collect_clients(params, segment_names)
+          fail "These are errors while collecting clients from input data:\n#{errors.join("\n")}" unless errors.empty?
 
           results = clients.map do |client|
             {
@@ -57,7 +58,6 @@ module GoodData
             }
           end
 
-          # Return results
           {
             results: results,
             params: {
@@ -72,8 +72,10 @@ module GoodData
           project_id_column = params.project_id_column || 'project_id'
           project_title_column = params.project_title_column || 'project_title'
           project_token_column = params.project_token_column || 'project_token'
+          client = params.gdc_gd_client
 
           clients = []
+          errors = []
           data_source = GoodData::Helpers::DataSource.new(params.input_source)
           input_data = File.open(data_source.realize(params), 'r:UTF-8')
           GoodData.logger.debug("Input data: #{input_data.read}")
@@ -83,10 +85,22 @@ module GoodData
             segment_name = row[segment_id_column]
             GoodData.logger.debug("Segment name: #{segment_name}")
             if segment_names.nil? || segment_names.include?(segment_name)
+              client_id = row[client_id_column]
+              pid = row[project_id_column]
+
+              if pid
+                begin
+                  errors << "Project #{pid} of client #{client_id} is deleted." if client.projects(pid).deleted?
+                rescue
+                  errors << "Seems like you (user executing the script - #{client.user.login}) \
+                  do not have access to project \"#{pid}\" of client \"#{client_id}\""
+                end
+              end
+
               clients << {
-                id: row[client_id_column],
+                id: client_id,
                 segment: segment_name,
-                project: row[project_id_column],
+                project: pid,
                 settings: [
                   {
                     name: 'lcm.token',
@@ -102,11 +116,11 @@ module GoodData
           end
 
           if clients.empty?
-            fail "No segments or clients qualify for provisioning. \
+            errors << "No segments or clients qualify for provisioning. \
             Please check the input source data, platform segments, and the SEGMENTS_FILTER parameter. \
             The intersection of these three elements is empty set."
           end
-          clients
+          [clients, errors]
         end
       end
     end