gooddata-edge 0.6.27.edge

data/gooddata

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require_relative "lib/gooddata/cli/cli"
+
+require_relative 'lib/gooddata/models/domain'
+
+res = GoodData::CLI.main(ARGV)
+
+exit(res)

data/gooddata.gemspec

@@ -0,0 +1,63 @@
+# encoding: utf-8
+$LOAD_PATH.push File.expand_path('../lib/', __FILE__)
+require 'gooddata/version'
+
+Gem::Specification.new do |s|
+  s.name = 'gooddata-edge'
+  s.version = GoodData::VERSION
+  s.licenses = ['BSD']
+
+  s.required_rubygems_version = Gem::Requirement.new('>= 0') if s.respond_to? :required_rubygems_version=
+  s.authors = [
+    'Pavel Kolesnikov',
+    'Thomas Watson Steen',
+    'Tomas Svarovsky',
+    'Tomas Korcak'
+  ]
+
+  s.summary = 'A convenient Ruby wrapper around the GoodData RESTful API'
+  s.description = 'Use the GoodData::Client class to integrate GoodData into your own application or use the CLI to work with GoodData directly from the command line.'
+  s.email = 'pavel@gooddata.com'
+  s.extra_rdoc_files = %w(LICENSE README.md)
+
+  s.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+
+  s.homepage = 'http://github.com/gooddata/gooddata-ruby'
+  s.require_paths = ['lib']
+
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'debase', '~> 0.1', '>= 0.1.7' if !ENV['TRAVIS_BUILD'] && RUBY_VERSION >= '2.0.0'
+  s.add_development_dependency 'license_finder', '~> 2.0', '>= 2.0.4'
+  s.add_development_dependency 'rake', '~> 10.4', '>= 10.4.2'
+  s.add_development_dependency 'rake-notes', '~> 0.2', '>= 0.2.0'
+  s.add_development_dependency 'redcarpet', '~> 3.1', '>= 3.1.1' if RUBY_PLATFORM != 'java'
+  s.add_development_dependency 'rspec', '~> 3.4', '>= 3.4.0'
+  s.add_development_dependency 'rspec-expectations', '~> 3.4', '>= 3.4.0'
+  s.add_development_dependency 'rubocop', '~> 0.34', '>= 0.34.0'
+  s.add_development_dependency 'ruby-debug-ide', '~> 0.4' if !ENV['TRAVIS_BUILD'] && RUBY_VERSION >= '2.0.0'
+  s.add_development_dependency 'simplecov', '~> 0.10', '>= 0.10.0'
+  s.add_development_dependency 'webmock', '~> 1.21', '>= 1.21.0'
+  s.add_development_dependency 'yard', '~> 0.8.7.6'
+  s.add_development_dependency 'yard-rspec', '~> 0.1'
+  s.add_development_dependency 'ZenTest', '~> 4.10', '>= 4.11.0'
+
+  #s.add_dependency 'aws-sdk', '~> 1.45'
+  s.add_dependency 'docile', '~> 1.1', '>= 1.1.5'
+  s.add_dependency 'erubis', '~> 2.7', '>= 2.7.0'
+  s.add_dependency 'gli', '~> 2.13', '>= 2.13.2'
+  s.add_dependency 'highline', '~> 1.7', '>= 1.7.3'
+  s.add_dependency 'json_pure', '~> 2', '>= 1.8.2'
+  s.add_dependency 'multi_json', '~> 1.11', '>= 1.11.2'
+  s.add_dependency 'parseconfig', '~> 1.0', '>= 1.0.6'
+  s.add_dependency 'pmap', '~> 1.0', '>= 1.0.2'
+  s.add_dependency 'pry'
+  s.add_dependency 'restforce', '~> 2.1', '>= 2.1.1'
+  s.add_dependency 'rest-client', '~> 1.8', '>= 1.8.0'
+  s.add_dependency 'rubyzip', '~> 1.1', '>= 1.1.7'
+  s.add_dependency 'salesforce_bulk_query-edge', '~> 0.2', '>= 0.2.1'
+  s.add_dependency 'terminal-table', '~> 1.5', '>= 1.5.2'
+  s.add_dependency 'aws-sdk', '~> 2.0'
+  s.add_dependency 'thread_safe'
+end

data/lib/gooddata.rb

@@ -0,0 +1,31 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'pmap'
+$pmap_default_thread_count = 20 # rubocop:disable GlobalVars
+
+# GoodData Module
+module GoodData
+end
+
+# Modules
+require_relative 'gooddata/bricks/bricks'
+require_relative 'gooddata/commands/commands'
+require_relative 'gooddata/core/core'
+require_relative 'gooddata/data/data'
+require_relative 'gooddata/exceptions/exceptions'
+require_relative 'gooddata/helpers/helpers'
+require_relative 'gooddata/models/models'
+
+# Files
+require_relative 'gooddata/app/app'
+require_relative 'gooddata/client'
+require_relative 'gooddata/connection'
+require_relative 'gooddata/extract'
+require_relative 'gooddata/version'
+
+# Extensions
+require_relative 'gooddata/extensions/extensions'

data/lib/gooddata/app/app.rb

@@ -0,0 +1,16 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative '../rest/rest'
+
+module GoodData
+  # Base class for Ruby SDK CLI Apps
+  class App
+    def main
+      fail NotImplementedError 'Application must implement #main'
+    end
+  end
+end

data/lib/gooddata/bricks/base_downloader.rb

@@ -0,0 +1,86 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'pathname'
+require 'aws-sdk'
+
+module GoodData
+  module Bricks
+    class BaseDownloader
+      def initialize(params)
+        @params = params
+        @logger = @params['GDC_LOGGER']
+      end
+
+      def pre_process(meta)
+        meta
+      end
+
+      def download
+        @logger.info 'would download data' if @logger
+        []
+      end
+
+      def backup(meta)
+        @logger.info 'would send a backup list of files to backup' if @logger
+
+        files = meta['objects'].map { |_k, o| o }.reduce([]) do |a, e|
+          a + e['filenames']
+        end
+
+        bucket_name = @params['s3_backup_bucket_name']
+
+        s3 = Aws::S3::Resource.new(
+          :access_key_id => @params['aws_access_key_id'],
+          :secret_access_key => @params['aws_secret_access_key']
+        )
+
+        bucket = s3.bucket(bucket_name)
+        bucket = s3.create_bucket(bucket: bucket_name) unless bucket.exists?
+
+        files.each do |file|
+          file_path = Pathname.new(file)
+          target_path = Pathname.new(@params['s3_backup_path'] || '') + file_path.basename
+          obj = bucket.object(target_path)
+          obj.put(body: file_path)
+          @logger.info "Backed up file #{file_path} to s3 #{target_path}" if @logger
+        end
+
+        meta
+      end
+
+      def post_process(meta)
+        @logger.info 'Maybe some postprocessing' if @logger
+        meta
+      end
+
+      # Run downloader
+      def run
+        downloaded_data = download
+        downloaded_data = pre_process(downloaded_data)
+        backup(downloaded_data) unless @params['skip_backup']
+
+        downloaded_data = post_process(downloaded_data)
+
+        # save the state - whatever is in the return value of #download in the :persist key .. to project metadata
+        if downloaded_data[:persist]
+          accumulated_state = downloaded_data[:persist].reduce([]) do |memo, item|
+            item.key?(:state) ? memo.concat(item[:state]) : memo
+          end
+          accumulated_state.each do |item|
+            key = item[:key]
+            val = item[:value]
+
+            @logger.info "Saving metadata #{key} => #{val}" if @logger
+            GoodData::ProjectMetadata[key] = val
+          end
+        end
+
+        post_process(downloaded_data)
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/brick.rb

@@ -0,0 +1,38 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_downloader'
+require_relative 'utils'
+
+require_relative 'middleware/middleware'
+
+module GoodData
+  module Bricks
+    # Brick base class
+    class Brick
+      def log(message)
+        logger = @params[:gdc_logger]
+        logger.info(message) unless logger.nil?
+      end
+
+      # Name of the brick
+      def name
+        self.class
+      end
+
+      # Version of brick, this should be implemented in subclasses
+      def version
+        fail NotImplementedError, 'Method version should be reimplemented'
+      end
+
+      # Bricks implementation which can be 'called'
+      def call(params = {})
+        @params = params
+        ''
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/bricks.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'pathname'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + '*.rb').each do |file|
+  require file
+end
+
+# Require all middleware
+require_relative 'middleware/middleware'

data/lib/gooddata/bricks/middleware/aws_middleware.rb

@@ -0,0 +1,29 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'aws-sdk'
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    class AWSMiddleware < Bricks::Middleware
+      def call(params)
+        params = params.to_hash
+        if params.key?('aws_client')
+          puts 'Setting up AWS-S3 connection'
+          fail "Unable to connect to AWS. Parameter \"aws_client\" seems to be empty" unless params['aws_client']
+          fail "Unable to connect to AWS. Parameter \"access_key_id\" is missing" if params['aws_client']['access_key_id'].blank?
+          fail "Unable to connect to AWS. Parameter \"secret_access_key\" is missing" if params['aws_client']['secret_access_key'].blank?
+          s3 = Aws::S3::Resource.new(params['aws_client'])
+          params['aws_client']['s3_client'] = s3
+          @app.call(params)
+        else
+          @app.call(params)
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/base_middleware.rb

@@ -0,0 +1,56 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'hashie'
+
+module GoodData
+  module Bricks
+    class Middleware
+      attr_accessor :app
+
+      include Bricks::Utils
+
+      # Loads defaults to params from a json file in @config.
+      #
+      # The idea is to have a set of parameter defaults
+      # for a middleware. The defaults are loaded from a json file.
+      # If a brick user wants to override a default, they can
+      # do that in runtime params which come to the method in 'params'.
+      #
+      # A deep merge is done on the params. Arrays and other
+      # non-hash types are overwritten (params win).
+      #
+      # ### Examples
+      #
+      # A brick developer develops a SalesforceDownloaderMiddleware
+      # with default preset 'gse' having a configuration preset
+      # {"entities": ["Acount", "Event", "OpportunityLineItem", "Opportunity", "User"]}
+      #
+      # The brick user only wants to use Opportunity, so he passes
+      # runtime parameter {"entities": ["Opportunity"]} which overrides
+      # the default.
+      # See spec/bricks/bricks_spec.rb for usage.
+      def load_defaults(params)
+        # if default params given, fill what's not given in runtime params
+        if @config
+          # load it from file and merge it
+          defaults = { 'config' => MultiJson.load(File.read(@config)) }
+          default_params = GoodData::Helpers::DeepMergeableHash[defaults]
+          params = default_params.deep_merge(params)
+        end
+        params
+      end
+
+      def call(params)
+        load_defaults(params)
+      end
+
+      def initialize(options = {})
+        @app = options[:app]
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/bench_middleware.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'benchmark'
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    class BenchMiddleware < Bricks::Middleware
+      def call(params)
+        params = params.to_hash
+        puts 'Starting timer'
+        result = nil
+        report = Benchmark.measure { result = @app.call(params) }
+        puts 'Stopping timer'
+        pp report
+        result
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb

@@ -0,0 +1,37 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'salesforce_bulk_query'
+
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    class BulkSalesforceMiddleware < Bricks::Middleware
+      DEFAULT_VERSION = '29.0'
+
+      def self.create_client(params)
+        salesforce = nil
+        if params['salesforce_client']
+
+          client = params['salesforce_client']
+          client.authenticate!
+
+          salesforce = SalesforceBulkQuery::Api.new(client, :logger => params['GDC_LOGGER'])
+          # SalesforceBulkQuery adds its own Restforce logging so turn it off
+          Restforce.log = false if params['GDC_LOGGER']
+        end
+        params.merge('salesforce_bulk_client' => salesforce)
+      end
+
+      def call(params)
+        params = params.to_hash
+        params = self.class.create_client(params)
+        @app.call(params)
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/decode_params_middleware.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    # Converts params from encoded hash to decoded hash
+    class DecodeParamsMiddleware < Bricks::Middleware
+      def call(params)
+        params = params.to_hash
+
+        @app.call(GoodData::Helpers.decode_params(params))
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/fs_download_middleware.rb

@@ -0,0 +1,48 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    class FsProjectDownloadMiddleware < Bricks::Middleware
+      def call(params)
+        params = params.to_hash
+        (params['gdc_files_to_download'] || []).each do |source|
+          case source[:type].to_s
+          when 'ads'
+            CSV.open(source[:to], 'w') do |csv|
+              header_written = false
+              header = nil
+              dwh = params['ads_client']
+              dwh.execute_select(source[:query]) do |row|
+                unless header_written
+                  header_written = true
+                  header = row.keys
+                  csv << header
+                end
+                csv << row.values_at(*header)
+              end
+            end
+          when 'staging'
+            webdav_uri = GoodData.project_webdav_path
+            dav = Net::DAV.new(webdav_uri, :curl => false)
+            dav.verify_server = false
+            dav.credentials(params['GDC_USERNAME'], params['GDC_PASSWORD'])
+            dav.find(path, recursive: true, suppress_errors: true) do |item|
+              puts 'Checking: ' + item.url.to_s
+              name = (item.uri - webdav_uri).to_s
+              File.open(name, 'w') do |f|
+                f << item.content
+              end
+            end
+          end
+        end
+        @app.call(params)
+      end
+    end
+  end
+end