gooddata-bricks 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f697a92c058c60827794ac8e144984d87621eae6
-  data.tar.gz: 5fbf6db7fd259423b9e859374c3e14b25e555974
+  metadata.gz: 1bbf4e721ee07561d0a148a71a03be11adbdc43f
+  data.tar.gz: 05d5f2febf699e6a10113c500ae83d1c396789e3
 SHA512:
-  metadata.gz: c4968620fa9e9536a14cb4f6c2b631fcfe7922500d124dbe7ac929c60e259f158b664b5fb2fbd05716614025d474b91c545b2abd2188a19305aeb7b2a2aa49da
-  data.tar.gz: db3b494e9b977b8b05ec15d113e44e43fa4a7fd8cdd06f03584e42ff62040149ac54d344bed19eb9d041edc3dc64abae2871d0b2a0d7a482a1e42aee20e1b77b
+  metadata.gz: 90a7c09c10fcad5d51419f2a16f056a2843b3b2524ff00e4bba5ef7487879b4c07b48dbee76d31953f7d6e0355acc6783cec43b4bf5d1be80e52e56358f30bbd
+  data.tar.gz: 515f0a2473d13895cab46eee334422b4e9af19282f69201613fb14c099c2f30bc062cea3704d388a044093309ecd937293cd2d88bbf496d9eb201eea97a8a2a7

data/.gitignore

@@ -1,3 +1,4 @@
+/.idea/
 /.bundle/
 /.yardoc
 /Gemfile.lock

data/.rubocop.yml

@@ -1,20 +1,95 @@
+AllCops:
+  DisplayCopNames: true
+
 AbcSize:
   Enabled: false
 
+ClassLength:
+  Enabled: false
+
+Metrics/BlockLength:
+  Max: 40
+
+ModuleLength:
+  Enabled: false
+
+CyclomaticComplexity:
+  Enabled: false
+
 Documentation:
   Enabled: false
 
+EachWithObject:
+  Enabled: false
+  
+Eval:
+  Enabled: false
+
+FormatString:
+  Enabled: false
+
+HashSyntax:
+  Enabled: false
+
 LineLength:
   Max: 323
 
 MethodLength:
   Enabled: false
 
-CyclomaticComplexity:
+PerceivedComplexity:
   Enabled: false
 
-PerceivedComplexity:
+Alias:
   Enabled: false
 
-ClassLength:
+Casecmp:
+  Enabled: false
+
+ConditionalAssignment:
+  Enabled: false
+
+GuardClause:
+  Enabled: false
+
+IdenticalConditionalBranches:
+  Enabled: false
+
+IfInsideElse:
+  Enabled: false
+
+IneffectiveAccessModifier:
+  Enabled: false
+
+MultilineMethodCallIndentation:
+  Enabled: false
+
+MutableConstant:
+  Enabled: false
+
+RedundantBlockCall:
+  Enabled: false
+
+RedundantMerge:
+  Enabled: false
+
+RedundantParentheses:
+  Enabled: false
+
+StringLiterals:
+  Enabled: false
+
+SignalException:
+  Enabled: false
+
+TimesMap:
+  Enabled: false
+
+UnneededDisable:
+  Enabled: false
+
+UnneededInterpolation:
+  Enabled: false
+
+UselessAccessModifier:
   Enabled: false

data/gooddata-bricks.gemspec

@@ -25,15 +25,15 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.10"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency 'rubocop', '~> 0.34', '>= 0.34.0'
+  spec.add_development_dependency "bundler", "~> 1.13", '>= 1.13.6'
+  spec.add_development_dependency "rake", "~> 11.1", '>= 11.1.1'
+  spec.add_development_dependency "rspec", '~> 3.5', '>= 3.5.0'
+  spec.add_development_dependency 'rubocop', '~> 0.45', '>= 0.45.0'
 
   spec.add_dependency 'aws-sdk', '~> 1.45'
   spec.add_dependency 'restforce', '~> 2.1', '>= 2.1.1'
   spec.add_dependency 'salesforce_bulk_query', '~> 0.2', '>= 0.2.0'
   spec.add_dependency 'gooddata'
   spec.add_dependency 'gooddata_datawarehouse'
-  spec.add_dependency 'multi_json'
+  spec.add_dependency 'multi_json', '~> 1.11', '>= 1.11.2'
 end

data/lib/gooddata/bricks/base_downloader.rb

@@ -0,0 +1,86 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'pathname'
+require 'aws'
+
+module GoodData
+  module Bricks
+    class BaseDownloader
+      def initialize(params)
+        @params = params
+        @logger = @params['GDC_LOGGER']
+      end
+
+      def pre_process(meta)
+        meta
+      end
+
+      def download
+        @logger.info 'would download data' if @logger
+        []
+      end
+
+      def backup(meta)
+        @logger.info 'would send a backup list of files to backup' if @logger
+
+        files = meta['objects'].map { |_k, o| o }.reduce([]) do |a, e|
+          a + e['filenames']
+        end
+
+        bucket_name = @params['s3_backup_bucket_name']
+
+        s3 = AWS::S3.new(
+          :access_key_id => @params['aws_access_key_id'],
+          :secret_access_key => @params['aws_secret_access_key']
+        )
+
+        bucket = s3.buckets[bucket_name]
+        bucket = s3.buckets.create(bucket_name) unless bucket.exists?
+
+        files.each do |file|
+          file_path = Pathname.new(file)
+          target_path = Pathname.new(@params['s3_backup_path'] || '') + file_path.basename
+          obj = bucket.objects[target_path]
+          obj.write(file_path)
+          @logger.info "Backed up file #{file_path} to s3 #{target_path}" if @logger
+        end
+
+        meta
+      end
+
+      def post_process(meta)
+        @logger.info 'Maybe some postprocessing' if @logger
+        meta
+      end
+
+      # Run downloader
+      def run
+        downloaded_data = download
+        downloaded_data = pre_process(downloaded_data)
+        backup(downloaded_data) unless @params['skip_backup']
+
+        downloaded_data = post_process(downloaded_data)
+
+        # save the state - whatever is in the return value of #download in the :persist key .. to project metadata
+        if downloaded_data[:persist]
+          accumulated_state = downloaded_data[:persist].reduce([]) do |memo, item|
+            item.key?(:state) ? memo.concat(item[:state]) : memo
+          end
+          accumulated_state.each do |item|
+            key = item[:key]
+            val = item[:value]
+
+            @logger.info "Saving metadata #{key} => #{val}" if @logger
+            GoodData::ProjectMetadata[key] = val
+          end
+        end
+
+        post_process(downloaded_data)
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/bench_middleware.rb

@@ -6,6 +6,7 @@
 
 require 'benchmark'
 require_relative 'base_middleware'
+require 'pp'
 
 module GoodData
   module Bricks

data/lib/gooddata/bricks/middleware/dwh_middleware.rb

@@ -12,7 +12,7 @@ module GoodData
     class WarehouseMiddleware < Bricks::Middleware
       def call(params)
         if params.key?('ads_client')
-          puts "Setting up ADS connection to #{params['ads_client']['ads_id']}"
+          GoodData.logger.info "Setting up ADS connection to #{params['ads_client']['ads_id'] || params['ads_client']['jdbc_url']}"
           raise "ADS middleware needs username either as part of ads_client spec or as a global 'GDC_USERNAME' parameter" unless params['ads_client']['username'] || params['GDC_USERNAME']
           raise "ADS middleware needs password either as part of ads_client spec or as a global 'GDC_PASSWORD' parameter" unless params['ads_client']['password'] || params['GDC_PASSWORD']
 

data/lib/gooddata/bricks/middleware/gooddata_middleware.rb

@@ -9,38 +9,77 @@ require_relative 'base_middleware'
 module GoodData
   module Bricks
     class GoodDataMiddleware < Bricks::Middleware
+      DEFAULT_PROTOCOL = 'https'
+      DEFAULT_HOSTNAME = 'secure.gooddata.com'
+
       def call(params)
+        # Convert possible jruby hash to plain hash
         params = params.to_hash
+
+        # Transform keys
+        params = GoodData::Helpers.deep_stringify_keys(params)
+        # params = GoodData::Helpers.deep_symbolize_keys(params)
+
+        # Set logger
         logger = params['GDC_LOGGER']
-        token_name = 'GDC_SST'
-        protocol_name = 'CLIENT_GDC_PROTOCOL'
-        server_name = 'CLIENT_GDC_HOSTNAME'
-        project_id = params['GDC_PROJECT_ID']
-        verify_ssl = params['GDC_VERIFY_SSL'] == 'false' || params['GDC_VERIFY_SSL'] == false ? false : true
-
-        server = if params[protocol_name] && params[server_name]
-                   "#{params[protocol_name]}://#{params[server_name]}"
-                 end
-
-        client = if params['GDC_USERNAME'].nil? || params['GDC_PASSWORD'].nil?
-                   puts "Connecting with SST to server #{server}"
-                   raise 'SST (SuperSecureToken) not present in params' if params[token_name].nil?
-                   GoodData.connect(sst_token: params[token_name], server: server, verify_ssl: verify_ssl)
-                 else
-                   puts "Connecting as #{params['GDC_USERNAME']} to server #{server}"
-                   GoodData.connect(params['GDC_USERNAME'], params['GDC_PASSWORD'], server: server, verify_ssl: verify_ssl)
-                 end
-        project = client.projects(project_id)
-        GoodData.project = project
         GoodData.logger = logger
-        returning_value = @app.call(params.merge('GDC_GD_CLIENT' => client, 'gdc_project' => project))
+
+        # Connect Client
+        client = GoodDataMiddleware.connect(
+          params['CLIENT_GDC_PROTOCOL'] || DEFAULT_PROTOCOL,
+          params['CLIENT_GDC_HOSTNAME'] || DEFAULT_HOSTNAME,
+          params['GDC_VERIFY_SSL'].to_b,
+          params['GDC_USERNAME'],
+          params['GDC_PASSWORD'],
+          params['GDC_SST']
+        )
+
+        opts = params['development_client']
+        if opts
+          development_client = GoodDataMiddleware.connect(
+            opts['protocol'] || DEFAULT_PROTOCOL,
+            opts['hostname'] || DEFAULT_HOSTNAME,
+            opts['verify_ssl'].to_b,
+            opts['username'],
+            opts['password'],
+            opts['sst']
+          )
+        else
+          development_client = client
+        end
+
+        new_params = {
+          'GDC_GD_CLIENT' => client,
+          'development_client' => development_client
+        }
+
+        if params['GDC_PROJECT_ID']
+          new_params['gdc_project'] = GoodData.project = client.projects(params['GDC_PROJECT_ID'])
+        end
+
+        returning_value = @app.call(params.merge(new_params))
         begin
           client.disconnect
         rescue
-          puts 'Tried to disconnect. Was unsuccesfull. Proceeding anyway.'
+          puts 'Tried to disconnect. Was unsuccessful. Proceeding anyway.'
         end
         returning_value
       end
+
+      class << self
+        def connect(protocol, hostname, verify_ssl, username, password, sst_token)
+          server = "#{protocol}://#{hostname}" if protocol && hostname
+
+          if username.nil? || password.nil?
+            puts "Connecting with SST to server #{server}"
+            raise 'SST (SuperSecureToken) not present in params' if sst_token.nil?
+            GoodData.connect(sst_token: sst_token, server: server, verify_ssl: verify_ssl)
+          else
+            puts "Connecting as #{username} to server #{server}"
+            GoodData.connect(username, password, server: server, verify_ssl: verify_ssl)
+          end
+        end
+      end
     end
   end
 end

data/lib/gooddata/bricks/middleware/params_inspect_middleware.rb

@@ -0,0 +1,21 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'logger'
+
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    class ParamsInspectMiddleware < Bricks::Middleware
+      def call(params)
+        inspect = params[:inspect_params] || params['inspect_params']
+        puts 'Inspecting Parameters ...'
+        puts JSON.pretty_generate(params) if inspect.to_b
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/version.rb

@@ -6,6 +6,6 @@
 
 module GoodData
   module Bricks
-    VERSION = '0.5.0'.freeze
+    VERSION = '0.6.0'.freeze
   end
 end

data/lib/gooddata/helpers/data_helper.rb

@@ -57,8 +57,7 @@ module GoodData
 
       def realize_query(params)
         query = DataSource.interpolate_sql_params(@options[:query], params)
-        dwh = params['ads_client']
-        raise "Data Source needs a client to ads to be able to query the storage but 'ads_client' is empty." unless dwh
+        dwh = params['ads_client'] || params[:ads_client] || raise("Data Source needs a client to ads to be able to query the storage but 'ads_client' is empty.")
         filename = Digest::SHA256.new.hexdigest(query)
         measure = Benchmark.measure do
           CSV.open(filename, 'w') do |csv|
@@ -74,7 +73,7 @@ module GoodData
             end
           end
         end
-        puts "Realizing SQL query \"#{query}\" took #{measure.real}"
+        GoodData.logger.info "Realizing SQL query \"#{query}\" took #{measure.real}"
         filename
       end
 

metadata

@@ -1,81 +1,99 @@
 --- !ruby/object:Gem::Specification
 name: gooddata-bricks
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Tomas Svarovsky
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-05 00:00:00.000000000 Z
+date: 2017-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.13'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.13.6
   name: bundler
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: 1.13.6
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.1'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 11.1.1
   name: rake
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '11.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 11.1.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.5.0
   name: rspec
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.5.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.34'
-    - - '>='
+        version: '0.45'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.34.0
+        version: 0.45.0
   name: rubocop
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.34'
-    - - '>='
+        version: '0.45'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.34.0
+        version: 0.45.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.45'
   name: aws-sdk
@@ -83,16 +101,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.45'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.1
   name: restforce
@@ -100,19 +118,19 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.0
   name: salesforce_bulk_query
@@ -120,16 +138,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: gooddata
@@ -137,13 +155,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: gooddata_datawarehouse
@@ -151,23 +169,29 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.11'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.11.2
   name: multi_json
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.11'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.11.2
 description: Provides some utils, simple framework and wraps some commond functionality.
 email:
 - svarovsky@gooddata.com
@@ -175,9 +199,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .rubocop.yml
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -185,6 +209,7 @@ files:
 - Rakefile
 - gooddata-bricks.gemspec
 - lib/gooddata/bricks.rb
+- lib/gooddata/bricks/base_downloader.rb
 - lib/gooddata/bricks/brick.rb
 - lib/gooddata/bricks/middleware/aws_middleware.rb
 - lib/gooddata/bricks/middleware/base_middleware.rb
@@ -197,6 +222,7 @@ files:
 - lib/gooddata/bricks/middleware/gooddata_middleware.rb
 - lib/gooddata/bricks/middleware/logger_middleware.rb
 - lib/gooddata/bricks/middleware/middleware.rb
+- lib/gooddata/bricks/middleware/params_inspect_middleware.rb
 - lib/gooddata/bricks/middleware/restforce_middleware.rb
 - lib/gooddata/bricks/middleware/stdout_middleware.rb
 - lib/gooddata/bricks/middleware/twitter_middleware.rb
@@ -215,19 +241,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.6.10
 signing_key:
 specification_version: 4
 summary: Simple library that simplifies development of GoodData Bricks
 test_files: []
-has_rdoc: