gooddata 2.1.2-java → 2.1.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cc69941e78a5cd2665e60eb7e0671e529e6e77ea92ea3a11393343a2e0da36c
-  data.tar.gz: a218c0ecf178af93fe5ef2754eee2bfc5a9574df676becb545c444636f70d184
+  metadata.gz: 4bb535b9b99905e2d580d47a3725c78beeb25ab9c5652ea9fa9d42bb7e8ac535
+  data.tar.gz: 3ecc2f9407584961ccc64ba0ebe4ce9fb6ace4aaaadb5e431da8137870700d5c
 SHA512:
-  metadata.gz: eaba126714070e966e781e7c9ff518b77663db63902ec5aab8e42e575a1eea4849bf31788545c88949a7f47d88892fb0e44457968eb3609befdbd526b3c96d02
-  data.tar.gz: c9187dfa5fbdfd449cd50dca2b00a34d54b7af494d084a823ff2dbb91547af3993df3db84b8029b718c6bcc25c2b276609a9e950e421b6de55609b5db075da2e
+  metadata.gz: 1dfbb53956a4b45312d80ccbe2ac7ca0e90adeb57f21faf5e944e8993cfc82becdd7b760ff2eb6df417bf0b7091b8bc4dfe8c463ec31ba8fc12ec22f0c28dac7
+  data.tar.gz: b6245d9b24f298414095891a2ec5d54499d6561450210ced013598a49cfd7412100821d883cfb4834654489e14f9b9c1886159d58d09e949f2cc09af87cc3072

data/CHANGELOG.md

@@ -1,4 +1,9 @@
 # GoodData Ruby SDK Changelog
+## 2.1.3
+ - TMA-1394: upload_data takes mapping of gd labels to csv columns into account
+ - TMA-1292: K8s load and smoke spec
+ - TMA-1259: support file as ADS replacement
+
 ## 2.1.2
  - TMA-1429 fixed method name
 

data/Dockerfile

@@ -11,19 +11,25 @@ LABEL git_commit=$GIT_COMMIT
 LABEL bricks_version=$BRICKS_VERSION
 
 # which is required by RVM
-RUN yum install -y curl which \
+RUN yum install -y curl which patch make git \
     && yum clean all \
     && rm -rf /var/cache/yum
 
-RUN gpg2 --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-RUN curl -sSL https://get.rvm.io | bash -s stable
-
 # Switch to directory with sources
 WORKDIR /src
-ENV HOME=/src
+
+RUN groupadd -g 1003 lcmuser && \
+    useradd -r -u 1003 -g lcmuser lcmuser && \
+    mkhomedir_helper lcmuser && \
+    chown lcmuser: /home && \
+    chown lcmuser: /src
+USER lcmuser
+
+RUN gpg2 --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+RUN curl -sSL https://get.rvm.io | bash -s stable
 
 # login shell is required by rvm
-RUN /bin/bash -l -c ". /usr/local/rvm/scripts/rvm && rvm install jruby-9.2.5.0 && gem update --system \
+RUN /bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && rvm install jruby-9.2.5.0 && gem update --system \
     && gem install bundler rake"
 
 ENV GOODDATA_RUBY_COMMIT=$GIT_COMMIT
@@ -35,6 +41,6 @@ ADD ./VERSION .
 ADD ./Gemfile .
 ADD ./gooddata.gemspec .
 
-RUN /bin/bash -l -c ". /usr/local/rvm/scripts/rvm && bundle install"
+RUN /bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && bundle install"
 
 CMD [ "./bin/help.sh" ]

data/SDK_VERSION

@@ -1 +1 @@
-2.1.2
+2.1.3

data/bin/hello_world.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-/bin/bash -l -c ". /usr/local/rvm/scripts/rvm && bundle exec ./bin/run_brick.rb hello_world_brick"
+/bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && bundle exec ./bin/run_brick.rb hello_world_brick"

data/bin/help.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-/bin/bash -l -c ". /usr/local/rvm/scripts/rvm && bundle exec ./bin/run_brick.rb help_brick"
+/bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && bundle exec ./bin/run_brick.rb help_brick"

data/bin/provision.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-/bin/bash -l -c ". /usr/local/rvm/scripts/rvm && bundle exec ./bin/run_brick.rb provisioning_brick"
+/bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && bundle exec ./bin/run_brick.rb provisioning_brick"

data/bin/release.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-/bin/bash -l -c ". /usr/local/rvm/scripts/rvm && bundle exec ./bin/run_brick.rb release_brick"
+/bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && bundle exec ./bin/run_brick.rb release_brick"

data/bin/rollout.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-/bin/bash -l -c ". /usr/local/rvm/scripts/rvm && bundle exec ./bin/run_brick.rb rollout_brick"
+/bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && bundle exec ./bin/run_brick.rb rollout_brick"

data/bin/run_smoke_tests.sh

@@ -2,4 +2,4 @@
 
 export LCM_BRICKS_IMAGE_TAG=$GOODDATA_RUBY_COMMIT
 export GD_ENV=staging
-/bin/bash -l -c ". /usr/local/rvm/scripts/rvm && bundle exec rake -f lcm.rake test:smoke"
+/bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && bundle exec rake -f lcm.rake test:smoke"

data/bin/test_projects_cleanup.rb

@@ -85,7 +85,7 @@ def clean_up!(client, force, days)
   delete_project_by_title(/LCM SPEC PROJECT/, projects, days, force)
   delete_project_by_title(/LCM spec Client With Conflicting LDM Changes/, projects, days, force)
   delete_project_by_title(/LCM spec master project/, projects, days, force)
-  delete_project_by_title(/users brick load test/, client, days, force)
+  delete_project_by_title(/users brick load test/, projects, days, force)
   delete_ads_by_title(/Development ADS/, client, days, force)
   delete_ads_by_title(/Production ADS/, client, days, force)
   delete_ads_by_title(/TEST ADS/, client, days, force)

data/bin/user_filters.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-/bin/bash -l -c ". /usr/local/rvm/scripts/rvm && bundle exec ./bin/run_brick.rb user_filters_brick"
+/bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && bundle exec ./bin/run_brick.rb user_filters_brick"

data/bin/users.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-/bin/bash -l -c ". /usr/local/rvm/scripts/rvm && bundle exec ./bin/run_brick.rb users_brick"
+/bin/bash -l -c ". /home/lcmuser/.rvm/scripts/rvm && bundle exec ./bin/run_brick.rb users_brick"

data/lcm.rake

@@ -3,7 +3,7 @@ require 'fileutils'
 require 'pathname'
 require 'rspec/core/rake_task'
 
-test_cases = %i[integration slow userprov load smoke]
+test_cases = %i[integration slow userprov load]
 
 # Schema for new Bricks.
 brick_info_schema = {
@@ -83,6 +83,20 @@ namespace :test do
     end
   end
 
+  desc 'Run smoke tests'
+  RSpec::Core::RakeTask.new(:smoke) do |task|
+    ENV['GD_LCM_SPEC_SEGMENTS_MULTIPLIER'] = '1'
+    ENV['GD_LCM_SPEC_WORKSPACES_MULTIPLIER'] = '1'
+
+    ENV['GD_LCM_SPEC_USER_COUNT'] = '1'
+    ENV['GD_LCM_SPEC_PROJECT_COUNT'] = '1'
+
+    ENV['VCR_ON'] = 'false'
+    ENV['GD_LCM_SMOKE_TEST'] = 'true'
+
+    task.pattern = 'spec/lcm/load/**/*_spec.rb'
+  end
+
   namespace :docker do
     test_cases.each do |t|
       desc "Run #{t} tests in Docker"

data/lib/gooddata/bricks/middleware/context_manager.rb

@@ -37,7 +37,7 @@ module GoodData
     end
 
     def time_from_action_start(now = Time.now)
-      fail_if_development 'No action is being profiled' unless action
+      GoodData::LCM2::Helpers.fail_if_development 'No action is being profiled' unless action
       (now - @action_start) * 1000
     end
 
@@ -47,7 +47,7 @@ module GoodData
     # @param [Logger] logger, logger that should log current context info
     # @param [Time] now, allows to specify exact time, when outer call was performed
     def start_action(next_action, logger = nil, now = Time.now)
-      fail_if_development 'An action is already being profiled' if action
+      GoodData::LCM2::Helpers.fail_if_development 'An action is already being profiled' if action
 
       self.action = next_action
       @action_start = now
@@ -59,7 +59,7 @@ module GoodData
     #
     # @param [Logger] logger, logger that should log current context info
     def end_action(logger = nil)
-      fail_if_development 'No matching action to start found' unless action
+      GoodData::LCM2::Helpers.fail_if_development 'No matching action to start found' unless action
 
       logger.info '' if logger
       self.status = :not_in_action

data/lib/gooddata/helpers/csv_helper.rb

@@ -5,6 +5,7 @@
 # LICENSE file in the root directory of this source tree.
 
 require 'csv'
+require 'fileutils'
 
 module GoodData
   module Helpers
@@ -38,6 +39,18 @@ module GoodData
           res
         end
 
+        # Read data from csv as an array of hashes with symbol keys and parsed integers
+        # @option filename String
+        def read_as_hash(filename)
+          res = []
+          return res unless File.exist? filename
+
+          CSV.parse(File.read(filename), headers: true, header_converters: :symbol, converters: :integer).map do |row|
+            res << row.to_hash
+          end
+          res
+        end
+
         # Write data to CSV
         # @option opts [String] :path File to write data to
         # @option opts [Array] :data Mandatory array of data to write
@@ -55,6 +68,20 @@ module GoodData
             end
           end
         end
+
+        # Ammend a hash to CSV in a smart manner
+        # @option filename String
+        # @option data Hash
+        def ammend_line(filename, data)
+          current_data = read_as_hash(filename)
+          data_to_write = (current_data << data).map(&:sort).map { |r| Hash[r] }
+          FileUtils.mkpath(filename.split('/')[0...-1].join('/'))
+          CSV.open(filename, 'w', write_headers: true, headers: data_to_write.first.keys) do |csv|
+            data_to_write.each do |d|
+              csv << d.values
+            end
+          end
+        end
       end
     end
   end

data/lib/gooddata/lcm/actions/collect_segment_clients.rb

@@ -16,7 +16,7 @@ module GoodData
         param :gdc_gd_client, instance_of(Type::GdClientType), required: true
 
         description 'ADS Client'
-        param :ads_client, instance_of(Type::AdsClientType), required: true
+        param :ads_client, instance_of(Type::AdsClientType), required: false
 
         description 'Segments to manage'
         param :segments, array_of(instance_of(Type::SegmentType)), required: true
@@ -26,6 +26,12 @@ module GoodData
 
         description 'DataProduct'
         param :data_product, instance_of(Type::GDDataProductType), required: false
+
+        description 'Organization Name'
+        param :organization, instance_of(Type::StringType), required: false
+
+        description 'Domain'
+        param :domain, instance_of(Type::StringType), required: false
       end
 
       RESULT_HEADER = [
@@ -47,12 +53,22 @@ module GoodData
 
             raise "Client(s) missing workspace: #{missing_project_clients.join(', ')}. Please make sure all clients have workspace." unless missing_project_clients.empty?
 
-            latest_master_id = GoodData::LCM2::Helpers.latest_master_project(
-              params.release_table_name,
-              params.ads_client,
-              segment.segment_id
-            )[:master_project_id]
-            latest_master = client.projects(latest_master_id)
+            domain_name = params.organization || params.domain
+            fail "Either organisation or domain has to be specified in params" unless domain_name
+
+            client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
+
+            if params.ads_client
+              latest_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
+                params.release_table_name,
+                params.ads_client,
+                segment.segment_id
+              )
+            else
+              latest_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment.segment_id)
+            end
+
+            latest_master = client.projects(latest_master[:master_project_id])
 
             # TODO: Check res.first.nil? || res.first[:master_project_id].nil?
             master_pid = latest_master.pid

data/lib/gooddata/lcm/actions/create_segment_masters.rb

@@ -25,7 +25,7 @@ module GoodData
         param :domain, instance_of(Type::StringType), required: false
 
         description 'ADS Client'
-        param :ads_client, instance_of(Type::AdsClientType), required: true
+        param :ads_client, instance_of(Type::AdsClientType), required: false
 
         description 'Segments to manage'
         param :segments, array_of(instance_of(Type::SegmentType)), required: true
@@ -71,7 +71,7 @@ module GoodData
             ads_output_stage_prefix = segment_in.ads_output_stage_prefix
 
             # Create master project Postgres
-            version = get_project_version(params, segment_id) + 1
+            version = get_project_version(params, domain_name, segment_id) + 1
 
             master_name = segment_in.master_name.gsub('#{version}', version.to_s)
 
@@ -150,12 +150,16 @@ module GoodData
           }
         end
 
-        def get_project_version(params, segment_id)
-          current_master = GoodData::LCM2::Helpers.latest_master_project(
-            params.release_table_name,
-            params.ads_client,
-            segment_id
-          )
+        def get_project_version(params, domain_name, segment_id)
+          if params.ads_client
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
+              params.release_table_name,
+              params.ads_client,
+              segment_id
+            )
+          else
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment_id)
+          end
           return 0 unless current_master
           current_master[:version].to_i
         end

data/lib/gooddata/lcm/actions/ensure_release_table.rb

@@ -13,7 +13,7 @@ module GoodData
 
       PARAMS = define_params(self) do
         description 'ADS Client'
-        param :ads_client, instance_of(Type::AdsClientType), required: true
+        param :ads_client, instance_of(Type::AdsClientType), required: false
 
         description 'Table Name'
         param :release_table_name, instance_of(Type::StringType), required: false
@@ -28,24 +28,32 @@ module GoodData
 
       class << self
         def call(params)
-          replacements = {
-            table_name: params.release_table_name || DEFAULT_TABLE_NAME
-          }
-
-          path = File.expand_path('../../data/create_lcm_release.sql.erb', __FILE__)
-          query = GoodData::Helpers::ErbHelper.template_file(path, replacements)
-
-          sql_result = params.ads_client.execute(query)
-
-          # TODO: Format
-          GoodData.logger.info(JSON.pretty_generate(sql_result))
-
-          [
-            {
-              table_name: replacements[:table_name],
-              status: 'ok'
+          if params.ads_client
+            replacements = {
+              table_name: params.release_table_name || DEFAULT_TABLE_NAME
             }
-          ]
+
+            path = File.expand_path('../data/create_lcm_release.sql.erb', __dir__)
+            query = GoodData::Helpers::ErbHelper.template_file(path, replacements)
+
+            sql_result = params.ads_client.execute(query)
+
+            # TODO: Format
+            GoodData.logger.info(JSON.pretty_generate(sql_result))
+
+            [
+              {
+                table_name: replacements[:table_name],
+                status: 'ok'
+              }
+            ]
+          else
+            [
+              {
+                status: 'ok'
+              }
+            ]
+          end
         end
       end
     end

data/lib/gooddata/lcm/actions/synchronize_clients.rb

@@ -31,7 +31,7 @@ module GoodData
         param :release_table_name, instance_of(Type::StringType), required: false
 
         description 'ADS Client'
-        param :ads_client, instance_of(Type::AdsClientType), required: true
+        param :ads_client, instance_of(Type::AdsClientType), required: false
 
         description 'Additional Hidden Parameters'
         param :additional_hidden_params, instance_of(Type::HashType), required: false
@@ -61,14 +61,18 @@ module GoodData
           end
 
           results = segments.map do |segment|
-            current_master = GoodData::LCM2::Helpers.latest_master_project(
-              params.release_table_name,
-              params.ads_client,
-              segment.segment_id
-            )[:master_project_id]
+            if params.ads_client
+              current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
+                params.release_table_name,
+                params.ads_client,
+                segment.segment_id
+              )
+            else
+              current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, segment.segment_id)
+            end
 
             # TODO: Check res.first.nil? || res.first[:master_project_id].nil?
-            master = client.projects(current_master)
+            master = client.projects(current_master[:master_project_id])
 
             segment.master_project = master
             segment.save

data/lib/gooddata/lcm/actions/update_release_table.rb

@@ -13,7 +13,7 @@ module GoodData
 
       PARAMS = define_params(self) do
         description 'ADS Client'
-        param :ads_client, instance_of(Type::AdsClientType), required: true
+        param :ads_client, instance_of(Type::AdsClientType), required: false
 
         description 'Client Used for Connecting to GD'
         param :gdc_gd_client, instance_of(Type::GdClientType), required: true
@@ -55,7 +55,7 @@ module GoodData
               table_name: params.release_table_name || DEFAULT_TABLE_NAME
             }
 
-            update_release_table(params, placeholders)
+            update_release_table(params, placeholders, domain_name)
 
             {
               segment: segment_id,
@@ -74,13 +74,17 @@ module GoodData
           result
         end
 
-        def update_release_table(params, placeholders)
-          path = File.expand_path('../../data/insert_into_lcm_release.sql.erb', __FILE__)
-          default_query = GoodData::Helpers::ErbHelper.template_file(path, placeholders)
-          temp_query = (params.query && params.query.insert) || default_query
-          query = replace_placeholders(temp_query, placeholders)
+        def update_release_table(params, placeholders, domain_id)
+          if params.ads_client
+            path = File.expand_path('../data/insert_into_lcm_release.sql.erb', __dir__)
+            default_query = GoodData::Helpers::ErbHelper.template_file(path, placeholders)
+            temp_query = (params.query && params.query.insert) || default_query
+            query = replace_placeholders(temp_query, placeholders)
 
-          params.ads_client.execute(query)
+            params.ads_client.execute(query)
+          else
+            GoodData::LCM2::Helpers.update_latest_master_to_nfs(domain_id, placeholders[:segment_id], placeholders[:master_project_id], placeholders[:version])
+          end
         end
       end
     end

data/lib/gooddata/lcm/helpers/release_table_helper.rb

@@ -8,9 +8,10 @@ module GoodData
   module LCM2
     class Helpers
       DEFAULT_TABLE_NAME = 'LCM_RELEASE'
+      DEFAULT_NFS_DIRECTORY = 'release-tables'
 
       class << self
-        def latest_master_project(release_table_name, ads_client, segment_id)
+        def latest_master_project_from_ads(release_table_name, ads_client, segment_id)
           replacements = {
             table_name: release_table_name || DEFAULT_TABLE_NAME,
             segment_id: segment_id
@@ -23,6 +24,24 @@ module GoodData
           sorted = res.sort_by { |row| row[:version] }
           sorted.last
         end
+
+        def latest_master_project_from_nfs(domain_id, segment_id)
+          data = GoodData::Helpers::Csv.read_as_hash(path_to_release_table_file(domain_id, segment_id))
+          data.sort_by { |master| master[:version] }
+              .reverse.first
+        end
+
+        def update_latest_master_to_nfs(domain_id, segment_id, master_pid, version)
+          GoodData::Helpers::Csv.ammend_line(
+            path_to_release_table_file(domain_id, segment_id),
+            master_project_id: master_pid,
+            version: version
+          )
+        end
+
+        def path_to_release_table_file(domain_id, segment_id)
+          [DEFAULT_NFS_DIRECTORY, domain_id, segment_id + '.csv'].join('/')
+        end
       end
     end
   end

data/lib/gooddata/models/model.rb

@@ -17,6 +17,7 @@ require 'fileutils'
 require 'multi_json'
 require 'open-uri'
 require 'zip'
+require 'csv'
 
 ##
 # Module containing classes that counter-part GoodData server-side meta-data
@@ -152,11 +153,9 @@ module GoodData
       # @return [Hash] Batch upload result
       def upload_multiple_data(data, project_blueprint, options = { :client => GoodData.connection, :project => GoodData.project })
         client, project = GoodData.get_client_and_project(options)
-
         project ||= GoodData.project
 
         manifest = {
-
           'dataSetSLIManifestList' => data.map do |d|
             mode = d[:options] && d[:options][:mode] ? d[:options][:mode] : options[:mode] || 'FULL'
             GoodData::Model::ToManifest.dataset_to_manifest(project_blueprint, d[:dataset], mode)
@@ -165,7 +164,6 @@ module GoodData
 
         csv_headers = []
 
-        # create a temporary zip file
         dir = Dir.mktmpdir
         begin
           Zip::File.open("#{dir}/upload.zip", Zip::File::CREATE) do |zip|
@@ -173,26 +171,27 @@ module GoodData
             zip.get_output_stream('upload_info.json') { |f| f.puts JSON.pretty_generate(manifest) }
 
             data.zip(manifest['dataSetSLIManifestList']).each do |item|
+              column_mapping = item[0][:options] ? item[0][:options][:column_mapping] : nil
               path = item[0][:data]
               path = item[0][:data].path if item[0][:data].respond_to? :path
               inline_data = !path.is_a?(String)
-              csv_header = nil
+
+              data_to_upload = inline_data ? path : File.open(path)
 
               filename = item[1]['dataSetSLIManifest']['file']
 
-              if inline_data
-                csv_header = path.first
-                zip.get_output_stream(filename) do |f|
-                  path.each do |row|
-                    f.puts row.to_csv
+              zip.get_output_stream(filename) do |file|
+                data_to_upload.each_with_index do |row, index|
+                  row = CSV.parse(row).first unless inline_data
+
+                  if index.zero?
+                    row.map! { |h| column_mapping.key(h) || h } if column_mapping
+                    csv_headers << row
                   end
+
+                  file.puts row.to_csv
                 end
-              else
-                csv_header = File.open(path, &:gets).chomp.split(',')
-                zip.add(filename, path)
               end
-
-              csv_headers << csv_header
             end
           end
 
@@ -240,6 +239,8 @@ module GoodData
           manifest_cols = parts.flatten.map { |c| c['columnName'] }
 
           # extract some human readable error message from the webdav file
+          csv_headers.map!(&:to_sym)
+          manifest_cols.map!(&:to_sym)
           manifest_extra = manifest_cols - csv_headers
           csv_extra = csv_headers - manifest_cols
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gooddata
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.1.3
 platform: java
 authors:
 - Pavel Kolesnikov
@@ -14,7 +14,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-03-24 00:00:00.000000000 Z
+date: 2019-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement