gooddata 0.6.53 → 0.6.54

data/lib/gooddata/lcm/actions/collect_data_product.rb

@@ -0,0 +1,62 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class CollectDataProduct < BaseAction
+      DESCRIPTION = 'Collect DataProduct to be used in the actions'
+
+      PARAMS = define_params(self) do
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+
+        description 'DataProduct to manage'
+        param :data_product, instance_of(Type::StringType), required: false
+      end
+
+      RESULT_HEADER = [
+        :data_product
+      ]
+
+      class << self
+        def call(params)
+          params = params.to_hash
+          client = params.gdc_gd_client
+          domain_name = params.organization || params.domain
+          domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
+
+          if params.key?(:data_product)
+            begin
+              data_product = domain.data_products(params.data_product)
+            rescue RestClient::BadRequest
+              fail 'the specified DataProduct does not exist'
+            end
+          else
+            params.gdc_logger.info 'Please specify the data_product parameter when using the brick - trying to find a fallback'
+            data_products = domain.data_products(:all)
+            fail 'No data_product parameter specified - unable to fallback to a default' if data_products.length > 1 || data_products.empty?
+            data_product = data_products.first
+          end
+
+          results = [
+            {
+              data_product: data_product
+            }
+          ]
+
+          {
+            results: results,
+            params: {
+              data_product: data_product
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb

@@ -0,0 +1,62 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class CollectDymanicScheduleParams < BaseAction
+      DESCRIPTION = 'Collect Dynamic Schedule Params'
+
+      PARAMS = define_params(self) do
+        description 'Schedule Title Column'
+        param :schedule_title_column, instance_of(Type::StringType), required: false
+
+        description 'Dynamic Params'
+        param :dynamic_params, instance_of(Type::HashType), required: false
+      end
+
+      class << self
+        def call(params)
+          return [] unless params.dynamic_params
+
+          schedule_title_column = params.schedule_title_column || 'schedule_title'
+          client_id_column = params.client_id_column || 'client_id'
+          param_name_column = params.param_name_column || 'param_name'
+          param_value_column = params.param_value_column || 'param_value'
+          results = []
+
+          input_source = params.dynamic_params.input_source
+          data_source = GoodData::Helpers::DataSource.new(input_source)
+          input_data = File.open(data_source.realize(params), 'r:UTF-8')
+          GoodData.logger.debug("Input data: #{input_data.read}")
+
+          schedule_params = {}
+
+          CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
+            GoodData.logger.debug("Processing row: #{row}")
+            results << row.to_hash
+
+            client_id = row[client_id_column] ? row[client_id_column] : :all_clients
+            schedule_name = row[schedule_title_column] ? row[schedule_title_column] : :all_schedules
+
+            schedule_params[client_id] ||= {}
+            schedule_params[client_id][schedule_name] ||= {}
+
+            schedule_params[client_id][schedule_name].merge!(row[param_name_column] => row[param_value_column])
+          end
+
+          {
+            results: results,
+            params: {
+              schedule_params: schedule_params
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/{collect_attrs.rb → collect_ldm_objects.rb}

@@ -8,8 +8,8 @@ require_relative 'base_action'
 
 module GoodData
   module LCM2
-    class CollectAttributes < BaseAction
-      DESCRIPTION = "Collect all attributes (include CAs) to transfer label type and drill path"
+    class CollectLdmObjects < BaseAction
+      DESCRIPTION = "Collect all objects in LDM: attributes (include CAs), facts, datasets"
 
       PARAMS = define_params(self) do
         description 'Development Client Used for Connecting to GD'
@@ -28,7 +28,7 @@ module GoodData
           synchronize = params.synchronize.pmap do |info|
             from = info.from
             from_project = development_client.projects(from) || fail("Invalid 'from' project specified - '#{from}'")
-            objects = (from_project.attributes.to_a + from_project.labels.to_a).map(&:uri)
+            objects = (from_project.attributes.to_a + from_project.labels.to_a + from_project.datasets.to_a + from_project.facts.to_a).map(&:uri)
 
             info[:transfer_uris] ||= []
             info[:transfer_uris] += objects

data/lib/gooddata/lcm/actions/collect_meta.rb

@@ -13,8 +13,11 @@ module GoodData
       with objects inside dashboards (reports, metrics ...) from development projects"
 
       PARAMS = define_params(self) do
-        description 'Production Tag Name'
-        param :production_tag, instance_of(Type::StringType), required: false
+        description 'Production Tag Names'
+        param :production_tags, array_of(instance_of(Type::StringType)), required: false
+
+        description 'Production Tag Names'
+        param :production_tag, instance_of(Type::StringType), required: false, deprecated: true, replacement: :production_tags
 
         description 'Development Client Used for Connecting to GD'
         param :development_client, instance_of(Type::GdClientType), required: true
@@ -42,7 +45,7 @@ module GoodData
             from_project = development_client.projects(from) || fail("Invalid 'from' project specified - '#{from}'")
 
             segment_tags = segments_to_tags[info.segment]
-            production_tags = Helpers.parse_production_tags(params.production_tag, segment_tags)
+            production_tags = Helpers.parse_production_tags(params.production_tags || params.production_tag, segment_tags)
 
             if transfer_all || production_tags.empty?
               old_dashboards = GoodData::Dashboard.all(

data/lib/gooddata/lcm/actions/collect_segment_clients.rb

@@ -43,7 +43,8 @@ module GoodData
 
           domain_name = params.organization || params.domain
           domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
-          domain_segments = domain.segments
+          data_product = params.data_product
+          domain_segments = domain.segments(:all, data_product)
 
           segments = params.segments.map do |seg|
             domain_segments.find do |s|

data/lib/gooddata/lcm/actions/collect_segments.rb

@@ -24,19 +24,18 @@ module GoodData
 
       class << self
         def call(params)
-          client = params.gdc_gd_client
-
-          domain_name = params.organization || params.domain
-          domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
-          domain_segments = domain.segments
+          data_product = params.data_product
+          data_product_segments = data_product.segments
+          params.gdc_logger.info("Domain segments: #{data_product_segments}")
 
           if params.segments_filter
-            domain_segments.select! do |segment|
+            params.gdc_logger.info("Segments filter: #{params.segments_filter}")
+            data_product_segments.select! do |segment|
               params.segments_filter.include?(segment.segment_id)
             end
           end
 
-          segments = domain_segments.pmap do |segment|
+          segments = data_product_segments.pmap do |segment|
             project = nil
 
             begin

data/lib/gooddata/lcm/actions/collect_tagged_objects.rb

@@ -19,8 +19,11 @@ module GoodData
         description 'Synchronization Info'
         param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
 
-        description 'Tag Name'
-        param :production_tag, instance_of(Type::StringType), required: false
+        description 'Production Tag Names'
+        param :production_tags, array_of(instance_of(Type::StringType)), required: false
+
+        description 'Production Tag Names'
+        param :production_tag, instance_of(Type::StringType), required: false, deprecated: true, replacement: :production_tags
 
         description 'Segments to search for segment-specific production tags'
         param :segments, array_of(instance_of(Type::SegmentType)), required: false
@@ -34,7 +37,7 @@ module GoodData
           results = []
           segments_to_tags = Helpers.segment_production_tags(params.segments)
           transfer_all = GoodData::Helpers.to_boolean(params.transfer_all)
-          return results unless params.production_tag || segments_to_tags.any? || transfer_all
+          return results unless params.production_tags || params.production_tag || segments_to_tags.any? || transfer_all
           development_client = params.development_client
 
           synchronize = params.synchronize.pmap do |info|
@@ -42,7 +45,7 @@ module GoodData
             from_project = development_client.projects(from) || fail("Invalid 'from' project specified - '#{from}'")
 
             segment_tags = segments_to_tags[info.segment]
-            production_tags = Helpers.parse_production_tags(params.production_tag, segment_tags)
+            production_tags = Helpers.parse_production_tags(params.production_tags || params.production_tag, segment_tags)
             objects = []
             if transfer_all
               vizs = MdObject.query('visualization', MdObject, client: development_client, project: from_project)

data/lib/gooddata/lcm/actions/create_segment_masters.rb

@@ -32,6 +32,9 @@ module GoodData
 
         description 'Table Name'
         param :release_table_name, instance_of(Type::StringType), required: false
+
+        description 'Project Environment'
+        param :project_environment, instance_of(Type::StringType), required: false
       end
 
       DEFAULT_TABLE_NAME = 'LCM_RELEASE'
@@ -45,7 +48,8 @@ module GoodData
 
           domain_name = params.organization || params.domain
           domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
-          domain_segments = domain.segments
+          data_product = params.data_product
+          domain_segments = domain.segments(:all, data_product)
 
           # TODO: Support for 'per segment' provisioning
           segments = params.segments
@@ -71,7 +75,7 @@ module GoodData
 
             # Create new master project
             params.gdc_logger.info "Creating master project - name: '#{master_name}' development_project: '#{development_pid}', segment: '#{segment_id}', driver: '#{driver}'"
-            project = client.create_project(title: master_name, auth_token: token, driver: driver == 'vertica' ? 'vertica' : 'Pg')
+            project = client.create_project(title: master_name, auth_token: token, driver: driver == 'vertica' ? 'vertica' : 'Pg', environment: params.project_environment)
 
             # Does segment exists? If not, create new one and set initial master
             if segment
@@ -79,7 +83,7 @@ module GoodData
               status = 'untouched'
             else
               params.gdc_logger.info "Creating segment #{segment_id}, master #{project.pid}"
-              segment = domain.create_segment(segment_id: segment_id, master_project: project)
+              segment = data_product.create_segment(segment_id: segment_id, master_project: project)
               segment.synchronize_clients
               segment_in[:is_new] = true
               status = 'created'

data/lib/gooddata/lcm/actions/ensure_data_product.rb

@@ -0,0 +1,53 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class EnsureDataProduct < BaseAction
+      DESCRIPTION = 'Creates the DataProduct if it does not exist yet'
+
+      PARAMS = define_params(self) do
+        description 'client to connect to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+
+        description 'DataProduct to ensure'
+        param :data_product, instance_of(Type::StringType), required: false
+      end
+
+      RESULT_HEADER = [
+        :data_product
+      ]
+
+      class << self
+        def call(params)
+          params = params.to_hash
+          client = params.gdc_gd_client
+          domain_name = params.organization || params.domain
+          domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
+
+          if params.key?(:data_product)
+            begin
+              data_product = domain.data_products(params.data_product)
+            rescue RestClient::BadRequest
+              params.gdc_logger.info "Can not find DataProduct #{params.data_product}, creating it instead"
+              data_product = domain.create_data_product(id: params.data_product)
+            end
+          end
+
+          {
+            results: [
+              {
+                data_product: data_product
+              }
+            ]
+          }
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/lcm/actions/ensure_technical_users_domain.rb

@@ -19,7 +19,11 @@ module GoodData
         param :organization, instance_of(Type::StringType), required: true
 
         description 'Technical users'
-        param :technical_user, array_of(instance_of(Type::StringType)), required: false
+        param :technical_user, array_of(instance_of(Type::StringType)),
+              required: false, deprecated: true, replacement: :technical_users
+
+        description 'Technical users'
+        param :technical_users, array_of(instance_of(Type::StringType)), required: false
       end
 
       RESULT_HEADER = [
@@ -36,7 +40,7 @@ module GoodData
           domain_name = params.organization || params.domain
           domain = client.domain(domain_name) || fail("Invalid domain name specified - #{domain_name}")
 
-          technical_users = (params.technical_user || []).uniq
+          technical_users = (params.technical_users || params.technical_user || []).uniq
           technical_users.pmap do |technical_user|
             domain_user = domain.users.find do |du|
               du.login == technical_user

data/lib/gooddata/lcm/actions/ensure_technical_users_project.rb

@@ -16,7 +16,11 @@ module GoodData
         param :gdc_gd_client, instance_of(Type::GdClientType), required: true
 
         description 'Technical users'
-        param :technical_user, array_of(instance_of(Type::StringType)), required: false
+        param :technical_user, array_of(instance_of(Type::StringType)),
+              required: false, deprecated: true, replacement: :technical_users
+
+        description 'Technical users'
+        param :technical_users, array_of(instance_of(Type::StringType)), required: false
 
         description 'Synchronization Info'
         param :synchronize, array_of(instance_of(Type::SynchronizationInfoType)), required: true, generated: true
@@ -36,7 +40,7 @@ module GoodData
         def call(params)
           client = params.gdc_gd_client
 
-          technical_users = params.technical_user || []
+          technical_users = params.technical_users || params.technical_user || []
           new_users = technical_users.map do |technical_user|
             {
               login: technical_user,
@@ -46,24 +50,32 @@ module GoodData
 
           results = params.synchronize.pmap do |synchronize_info|
             synchronize_info[:to].pmap do |entry|
-              project = client.projects(entry[:pid])
-              res = project.create_users(new_users)
-
-              new_users.zip(res).map do |f, s|
-                {
-                  project: project.title,
-                  pid: project.pid,
-                  login: f[:login],
-                  role: f[:role],
-                  result: s[:type],
-                  message: s[:message],
-                  url: s[:user]
-                }
-              end
+              ensure_users(client, entry[:pid], new_users)
             end
-          end
+          end.flatten
+
+          results += params.clients.pmap { |input_client| input_client[:project] }.compact.pmap { |pid| ensure_users(client, pid, new_users) }.flatten if params.clients
+
+          results
+        end
+
+        private
+
+        def ensure_users(client, project_id, new_users)
+          project = client.projects(project_id)
+          res = project.create_users(new_users)
 
-          results.flatten
+          new_users.zip(res).map do |f, s|
+            {
+              project: project.title,
+              pid: project.pid,
+              login: f[:login],
+              role: f[:role],
+              result: s[:type],
+              message: s[:message],
+              url: s[:user]
+            }
+          end
         end
       end
     end

data/lib/gooddata/lcm/actions/execute_schedules.rb

@@ -0,0 +1,128 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class ExecuteSchedules < BaseAction
+      DESCRIPTION = 'Execute schedules'
+
+      PARAMS = define_params(self) do
+        description 'Client Used for Connecting to GD'
+        param :gdc_gd_client, instance_of(Type::GdClientType), required: true
+
+        description 'List Of Modes'
+        param :list_of_modes, instance_of(Type::StringType), required: true
+
+        description 'Work Done Identificator'
+        param :work_done_identificator, instance_of(Type::StringType), required: true
+
+        description 'Number Of Schedules In Batch'
+        param :number_of_schedules_in_batch, instance_of(Type::StringType), required: false, default: '1000'
+
+        description 'Delay Between Batches'
+        param :delay_between_batches, instance_of(Type::StringType), required: false, default: '0'
+
+        description 'Control Parameter'
+        param :control_parameter, instance_of(Type::StringType), required: false, default: 'MODE'
+
+        description 'Wait For Schedule'
+        param :wait_for_schedule, instance_of(Type::StringType), required: false, default: 'false'
+
+        description 'Segment List'
+        param :segment_list, instance_of(Type::StringType), required: false
+
+        description 'Domain'
+        param :domain, instance_of(Type::StringType), required: false
+
+        description 'Number Of Threads'
+        param :number_of_threads, instance_of(Type::StringType), required: false, default: '5'
+      end
+
+      class << self
+        def call(params)
+          logger = params.gdc_logger
+
+          client = params.gdc_gd_client
+          project = client.projects(params.gdc_project) || client.projects(params.gdc_project_id)
+
+          list_of_modes = params.list_of_modes.split('|').map(&:strip)
+          work_done_identificator = params.work_done_identificator
+          number_of_schedules_in_batch = Integer(params.number_of_schedules_in_batch || "1000")
+          delay_between_batches = Integer(params.delay_between_batches || "0")
+          control_parameter = params.control_parameter || "MODE"
+          wait = GoodData::Helpers.to_boolean(params.wait_for_schedule)
+
+          segment_list = params.segment_list
+          domain_name  = params.domain
+          fail "In case that you are using SEGMENT_LIST parameter, you need to fill out DOMAIN parameter" if !segment_list.nil? && domain_name.nil?
+
+          number_of_threads = Integer(params.number_of_threads || '5')
+
+          # The WORK_DONE_IDENTIFICATOR is flag which tells the executor to execute the schedules
+          # It could have special value IGNORE. In this case all corresponding schedules will be started during every run of this brick
+
+          start_schedules = work_done_identificator == 'IGNORE' || GoodData::Helpers.to_boolean(project.metadata[work_done_identificator])
+
+          project_list = []
+          if !segment_list.nil? && segment_list != ''
+            domain = client.domain(domain_name)
+            segment_list.split('|').each do |segment_identification|
+              segment = domain.segments(segment_identification)
+              project_list += segment.clients.map { |c| !c.project.nil? ? c.project.obj_id : nil }.compact
+            end
+            project_list.uniq!
+          end
+
+          if start_schedules
+            schedules_to_filter = client.projects.pmapcat do |p|
+              begin
+                p.schedules
+              rescue => e
+                logger.info "The retrieval of project schedules, for project #{p.obj_id} has failed. Message: #{e.message}."
+                []
+              end
+            end
+
+            schedules_to_start = schedules_to_filter.select { |schedule| list_of_modes.include?(schedule.params[control_parameter]) }
+
+            if !segment_list.nil? && segment_list != ''
+              schedules_to_start.delete_if { |schedule| !project_list.include?(schedule.project.obj_id) }
+            end
+
+            logger.info "Found #{schedules_to_start.count} schedules to execute"
+
+            schedules_to_start.each_slice(number_of_schedules_in_batch).each_with_index do |batch_schedules, batch_index|
+              batch_number = batch_index + 1
+              logger.info "Starting batch number #{batch_number}. Number of schedules in batch #{batch_schedules.count}."
+              batch_schedules.peach(number_of_threads) do |schedule|
+                begin
+                  tries ||= 5
+                  logger.info "Starting schedule for project #{schedule.project.pid} - #{schedule.project.title}. Schedule ID is #{schedule.obj_id}"
+                  schedule.execute(wait: wait)
+                rescue => e
+                  if (tries -= 1) > 0
+                    logger.warn "There was error during operation: #{e.message}. Retrying"
+                    sleep(5)
+                    retry
+                  else
+                    logger.info "We could not start schedule for project #{schedule.project.pid} - #{schedule.project.title} - #{e.message}"
+                  end
+                else
+                  logger.info "Operation finished"
+                end
+              end
+              logger.info "Entering sleep mode for #{delay_between_batches} seconds"
+              sleep(delay_between_batches)
+            end
+            project.set_metadata(work_done_identificator, 'false') unless work_done_identificator == 'IGNORE'
+          end
+        end
+      end
+    end
+  end
+end