gooddata 1.0.0-java

data/lib/gooddata/bricks/middleware/base_middleware.rb

@@ -0,0 +1,57 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'hashie'
+require 'multi_json'
+
+module GoodData
+  module Bricks
+    class Middleware
+      attr_accessor :app
+
+      include Bricks::Utils
+
+      # Loads defaults to params from a json file in @config.
+      #
+      # The idea is to have a set of parameter defaults
+      # for a middleware. The defaults are loaded from a json file.
+      # If a brick user wants to override a default, they can
+      # do that in runtime params which come to the method in 'params'.
+      #
+      # A deep merge is done on the params. Arrays and other
+      # non-hash types are overwritten (params win).
+      #
+      # ### Examples
+      #
+      # A brick developer develops a SalesforceDownloaderMiddleware
+      # with default preset 'gse' having a configuration preset
+      # {"entities": ["Acount", "Event", "OpportunityLineItem", "Opportunity", "User"]}
+      #
+      # The brick user only wants to use Opportunity, so he passes
+      # runtime parameter {"entities": ["Opportunity"]} which overrides
+      # the default.
+      # See spec/bricks/bricks_spec.rb for usage.
+      def load_defaults(params)
+        # if default params given, fill what's not given in runtime params
+        if @config
+          # load it from file and merge it
+          defaults = { 'config' => MultiJson.load(File.read(@config)) }
+          default_params = GoodData::Helpers::DeepMergeableHash[defaults]
+          params = default_params.deep_merge(params)
+        end
+        params
+      end
+
+      def call(params)
+        load_defaults(params)
+      end
+
+      def initialize(options = {})
+        @app = options[:app]
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/bench_middleware.rb

@@ -0,0 +1,25 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'benchmark'
+require_relative 'base_middleware'
+require 'pp'
+
+module GoodData
+  module Bricks
+    class BenchMiddleware < Bricks::Middleware
+      def call(params)
+        params = params.to_hash
+        puts 'Starting timer'
+        result = nil
+        report = Benchmark.measure { result = @app.call(params) }
+        puts 'Stopping timer'
+        pp report
+        result
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb

@@ -0,0 +1,37 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'salesforce_bulk_query'
+
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    class BulkSalesforceMiddleware < Bricks::Middleware
+      DEFAULT_VERSION = '29.0'.freeze
+
+      def self.create_client(params)
+        salesforce = nil
+        if params['salesforce_client']
+
+          client = params['salesforce_client']
+          client.authenticate!
+
+          salesforce = SalesforceBulkQuery::Api.new(client, logger: params['GDC_LOGGER'])
+          # SalesforceBulkQuery adds its own Restforce logging so turn it off
+          Restforce.log = false if params['GDC_LOGGER']
+        end
+        params.merge('salesforce_bulk_client' => salesforce)
+      end
+
+      def call(params)
+        params = params.to_hash
+        params = self.class.create_client(params)
+        @app.call(params)
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/decode_params_middleware.rb

@@ -0,0 +1,21 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    # Converts params from encoded hash to decoded hash
+    class DecodeParamsMiddleware < Bricks::Middleware
+      def call(params)
+        params = params.to_hash
+
+        # Decode and resolve reference parameters in gd_encoded_params
+        @app.call(GoodData::Helpers.decode_params(params, resolve_reference_params: true, convert_pipe_delimited_params: true))
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/dwh_middleware.rb

@@ -0,0 +1,41 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_middleware'
+
+require 'gooddata_datawarehouse' if RUBY_PLATFORM == 'java'
+
+module GoodData
+  module Bricks
+    # Connects to the warehouse (ADS) and enriches parameters with GoodData::Datawarehouse
+    class WarehouseMiddleware < Bricks::Middleware
+      def call(params)
+        if params.key?('ads_client') && (RUBY_PLATFORM == 'java')
+          GoodData.logger.info "Setting up ADS connection to #{params['ads_client']['ads_id'] || params['ads_client']['jdbc_url']}"
+          unless params['ads_client']['username'] || params['GDC_USERNAME']
+            raise "ADS middleware needs username either as part of " \
+                  "ads_client spec or as a global 'GDC_USERNAME' parameter"
+          end
+
+          unless params['ads_client']['password'] || params['GDC_PASSWORD']
+            raise "ADS middleware needs password either as part of " \
+                  "ads_client spec or as a global 'GDC_PASSWORD' parameter"
+          end
+
+          ads = GoodData::Datawarehouse.new(
+            params['ads_client']['username'] || params['GDC_USERNAME'],
+            params['ads_client']['password'] || params['GDC_PASSWORD'],
+            params['ads_client']['ads_id'],
+            jdbc_url: params['ads_client']['jdbc_url']
+          )
+          @app.call(params.merge('ads_client' => ads, :ads_client => ads))
+        else
+          @app.call(params)
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/fs_download_middleware.rb

@@ -0,0 +1,48 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    class FsProjectDownloadMiddleware < Bricks::Middleware
+      def call(params)
+        params = params.to_hash
+        (params['gdc_files_to_download'] || []).each do |source|
+          case source[:type].to_s
+          when 'ads'
+            CSV.open(source[:to], 'w') do |csv|
+              header_written = false
+              header = nil
+              dwh = params['ads_client']
+              dwh.execute_select(source[:query]) do |row|
+                unless header_written
+                  header_written = true
+                  header = row.keys
+                  csv << header
+                end
+                csv << row.values_at(*header)
+              end
+            end
+          when 'staging'
+            webdav_uri = GoodData.project_webdav_path
+            dav = Net::DAV.new(webdav_uri, curl: false)
+            dav.verify_server = false
+            dav.credentials(params['GDC_USERNAME'], params['GDC_PASSWORD'])
+            dav.find(path, recursive: true, suppress_errors: true) do |item|
+              puts 'Checking: ' + item.url.to_s
+              name = (item.uri - webdav_uri).to_s
+              File.open(name, 'w') do |f|
+                f << item.content
+              end
+            end
+          end
+        end
+        @app.call(params)
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/fs_upload_middleware.rb

@@ -0,0 +1,36 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    class FsProjectUploadMiddleware < Bricks::Middleware
+      def initialize(options = {})
+        super
+        @destination = options[:destination]
+      end
+
+      def call(params)
+        params = params.to_hash
+        returning(@app.call(params)) do |_|
+          destination = @destination
+          (params['gdc_files_to_upload'] || []).each do |f|
+            path = f[:path]
+            case destination.to_sym
+            when :staging
+              GoodData.client.get '/gdc/account/token', dont_reauth: true
+              url = GoodData.project_webdav_path
+              GoodData.upload_to_project_webdav(path)
+              puts "Uploaded local file \"#{path}\" to url \"#{url + path}\""
+            end
+          end
+        end
+      end
+    end
+    FsUploadMiddleware = FsProjectUploadMiddleware
+  end
+end

data/lib/gooddata/bricks/middleware/gooddata_middleware.rb

@@ -0,0 +1,112 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    # Connects to platform and enriches parameters with GoodData::Client
+    class GoodDataMiddleware < Bricks::Middleware
+      DEFAULT_PROTOCOL = 'https'
+      DEFAULT_HOSTNAME = 'secure.gooddata.com'
+
+      def call(params)
+        # Convert possible jruby hash to plain hash
+        params = params.to_hash
+
+        # Transform keys
+        params = GoodData::Helpers.stringify_keys(params)
+        # params = GoodData::Helpers.symbolize_keys(params)
+
+        # Set logger
+        logger = params['GDC_LOGGER']
+        GoodData.logger = logger
+
+        # Set parallelism
+        max_concurrency = params['max_concurrency'] || params['MAX_CONCURRENCY']
+        if max_concurrency && max_concurrency.to_i > 0
+          $pmap_default_thread_count = max_concurrency.to_i # rubocop:disable GlobalVars
+        end
+
+        # Connect Client
+        protocol = params['CLIENT_GDC_PROTOCOL'] || params['GDC_PROTOCOL'] || DEFAULT_PROTOCOL
+        hostname = params['CLIENT_GDC_HOSTNAME'] || params['GDC_HOSTNAME'] || DEFAULT_HOSTNAME
+        server = "#{protocol}://#{hostname}"
+        client = GoodDataMiddleware.connect(
+          server,
+          params['GDC_VERIFY_SSL'].to_b,
+          params['GDC_USERNAME'],
+          params['GDC_PASSWORD'],
+          params['GDC_SST']
+        )
+
+        opts = params['development_client']
+        if opts
+          if opts['server']
+            server = opts['server']
+          else
+            protocol = opts['protocol'] || DEFAULT_PROTOCOL
+            hostname = opts['hostname'] || DEFAULT_HOSTNAME
+            server = "#{protocol}://#{hostname}"
+          end
+
+          development_client = GoodDataMiddleware.connect(
+            server,
+            opts['verify_ssl'].to_b,
+            opts['username'] || opts['login'] || opts['email'],
+            opts['password'],
+            opts['sst']
+          )
+        else
+          development_client = client
+        end
+
+        new_params = {
+          'GDC_GD_CLIENT' => client,
+          'development_client' => development_client
+        }
+
+        if params['GDC_PROJECT_ID']
+          new_params['gdc_project'] = GoodData.project = client.projects(params['GDC_PROJECT_ID'])
+        end
+
+        returning_value = @app.call(params.merge(new_params))
+
+        # Try to disconnect client
+        begin
+          client.disconnect
+        rescue
+          puts 'Tried to disconnect client. Was unsuccessful. Proceeding anyway.'
+        end
+
+        # Try to disconnect development_client
+        begin
+          development_client.disconnect if development_client != client
+        rescue
+          puts 'Tried to disconnect development_client. Was unsuccessful. Proceeding anyway.'
+        end
+
+        returning_value
+      end
+
+      class << self
+        def connect(server, verify_ssl, username, password, sst_token) # rubocop:disable Metrics/ParameterLists
+          if username.nil? || password.nil?
+            puts "Connecting with SST to server #{server}"
+            raise 'SST (SuperSecureToken) not present in params' if sst_token.nil?
+            conn = GoodData.connect(sst_token: sst_token, server: server, verify_ssl: verify_ssl)
+          else
+            puts "Connecting as #{username} to server #{server}"
+            conn = GoodData.connect(username, password, server: server, verify_ssl: verify_ssl)
+          end
+          conn.stats_on
+
+          conn
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/logger_middleware.rb

@@ -0,0 +1,33 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'logger'
+
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    class LoggerMiddleware < Bricks::Middleware
+      def call(params)
+        params = params.to_hash
+        logger = nil
+        if params['GDC_LOGGING_OFF']
+          logger = NilLogger.new
+        else
+          logger = params[:GDC_LOGGER_FILE].nil? ? Logger.new(STDOUT) : Logger.new(params[:GDC_LOGGER_FILE])
+          logger.level = params['GDC_LOG_LEVEL'] if params['GDC_LOG_LEVEL']
+          logger.info('Pipeline starts')
+        end
+        params['GDC_LOGGER'] = logger
+        GoodData.logging_http_on if params['HTTP_LOGGING'] && params['HTTP_LOGGING'].to_b
+
+        returning(@app.call(params)) do |_result|
+          logger.info('Pipeline ending')
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/middleware.rb

@@ -0,0 +1,12 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'pathname'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob("#{base}/*_middleware.rb").each do |file|
+  require file
+end

data/lib/gooddata/bricks/middleware/restforce_middleware.rb

@@ -0,0 +1,58 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    class RestForceMiddleware < Bricks::Middleware
+      DEFAULT_VERSION = '29.0'.freeze
+
+      def self.prepare_credentials(downloader_config)
+        username = downloader_config['username']
+        password = downloader_config['password']
+        token = downloader_config['token']
+        oauth_refresh_token = downloader_config['oauth_refresh_token']
+
+        if username && password && token
+          {
+            username: username,
+            password: password,
+            security_token: token
+          }
+        elsif oauth_refresh_token && !oauth_refresh_token.empty?
+          {
+            refresh_token: oauth_refresh_token
+          }
+        end
+      end
+
+      def self.create_client(params)
+        downloader_config = params['config']['downloader']['salesforce']
+        credentials = prepare_credentials(params)
+
+        client = if credentials
+                   credentials[:client_id] = downloader_config['client_id']
+                   credentials[:client_secret] = downloader_config['client_secret']
+
+                   host = downloader_config['host']
+                   credentials[:host] = host if host
+                   credentials[:api_version] = downloader_config['api_version'] || DEFAULT_VERSION
+
+                   Restforce.log = true if params['GDC_LOGGER']
+
+                   Restforce.new(credentials)
+                 end
+        params.merge('salesforce_client' => client)
+      end
+
+      def call(params)
+        params = self.class.create_client(params)
+        @app.call(params)
+      end
+    end
+  end
+end