gooddata-bricks 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19b98bdc50b16bdb85db7a9698794eba74f660ce
-  data.tar.gz: 826ca8b4ca96c71eca424f6dbfa17cc167a21843
+  metadata.gz: c449efac4ebc29bbe1894ddacd889d6c5bc408e0
+  data.tar.gz: 1ddd5f72330bbcbd4837b3790f4c620d1a87e254
 SHA512:
-  metadata.gz: 3173fbe9fca5a422161d3d1a3e4e8d681bbad10c51d27845a4a242030237a7237b0de74273d2bdcc7a79146578b2025e86289bd5a9cb496ee07ecb95da72147b
-  data.tar.gz: 5d18283393e0548b0845cb6bb5e0b708d02ce54f40841fd106afd185be6b8396cf1c328a48691b0da67d74a88e27a7a6f7022ab4456b4500e2f6cc8112783961
+  metadata.gz: 0f46c3b8817ec7f367e4e45b629d3765541acbb875f3f059daa7634302ea9980e1032b818e94cb2d6575e256da5656dba4accff1cbf51ded0328b2f0f10fcc35
+  data.tar.gz: 1c9c0c4fa49f22ee8771ed314e534aef03a13de6005a2524026772c2f9b50d509ca76ee9971f49e5c8d9ed1a4086a4e5b5521adaa552c41e7c08dd991a8818cd

data/gooddata-bricks.gemspec

@@ -34,5 +34,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'restforce', '~> 2.1', '>= 2.1.1'
   spec.add_dependency 'salesforce_bulk_query', '~> 0.2', '>= 0.2.0'
   spec.add_dependency 'gooddata'
+  spec.add_dependency 'gooddata_datawarehouse'
   spec.add_dependency 'multi_json'
 end

data/lib/gooddata/bricks.rb

@@ -6,3 +6,4 @@
 
 require 'gooddata/bricks/version'
 require_relative 'bricks/brick'
+require_relative 'helpers/data_helper'

data/lib/gooddata/bricks/middleware/dwh_middleware.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_middleware'
+require 'gooddata_datawarehouse'
+
+module GoodData
+  module Bricks
+    class WarehouseMiddleware < Bricks::Middleware
+      def call(params)
+        if params.key?('ads_client')
+          puts "Setting up ADS connection to #{params['ads_client']['ads_id']}"
+          fail "ADS middleware needs username either as part of ads_client spec or as a global 'GDC_USERNAME' parameter" unless params['ads_client']['username'] || params['GDC_USERNAME']
+          fail "ADS middleware needs password either as part of ads_client spec or as a global 'GDC_PASSWORD' parameter" unless params['ads_client']['password'] || params['GDC_PASSWORD']
+          
+          ads = GoodData::Datawarehouse.new(params['ads_client']['username'] || params['GDC_USERNAME'], params['ads_client']['password'] || params['GDC_PASSWORD'], params['ads_client']['ads_id'])
+          @app.call(params.merge('ads_client' => ads, :ads_client => ads))
+        else
+          @app.call(params)
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/version.rb

@@ -6,6 +6,6 @@
 
 module GoodData
   module Bricks
-    VERSION = '0.1.0'.freeze
+    VERSION = '0.2.0'.freeze
   end
 end

data/lib/gooddata/helpers/data_helper.rb

@@ -0,0 +1,116 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'csv'
+require 'digest'
+require 'open-uri'
+
+module GoodData
+  module Helpers
+    class DataSource
+      attr_reader :realized
+
+      def initialize(opts = {})
+        opts = opts.is_a?(String) ? { type: :staging, path: opts } : opts
+        opts = GoodData::Helpers.symbolize_keys(opts)
+        @source = opts[:type]
+        @options = opts
+        @realized = false
+      end
+
+      def realize(params = {})
+        @realized = true
+        source = @source && @source.to_s
+        case source
+        when 'ads'
+          realize_query(params)
+        when 'staging'
+          realize_staging(params)
+        when 'web'
+          realize_link
+        when 's3'
+          realize_s3(params)
+        else
+          fail "DataSource does not support type \"#{source}\""
+        end
+      end
+
+      def realized?
+        @realized == true
+      end
+
+      private
+
+      def realize_query(params)
+        query = @options[:query]
+        dwh = params['ads_client']
+        fail "Data Source needs a client to ads to be able to query the storage but 'ads_client' is empty." unless dwh
+        filename = Digest::SHA256.new.hexdigest(query)
+        measure = Benchmark.measure do
+          CSV.open(filename, 'w') do |csv|
+            header_written = false
+            header = nil
+
+            dwh.execute_select(query) do |row|
+              unless header_written
+                header_written = true
+                header = row.keys
+                csv << header
+              end
+              csv << row.values_at(*header)
+            end
+          end
+        end
+        puts "Realizing SQL query \"#{query}\" took #{measure.real}"
+        filename
+      end
+
+      def realize_staging(params)
+        path = @options[:path]
+        url = URI.parse(path)
+        filename = Digest::SHA256.new.hexdigest(path)
+        if url.relative?
+          params['gdc_project'].download_file(path, filename)
+        else
+          params['GDC_GD_CLIENT'].download_file(path, filename)
+        end
+        filename
+      end
+
+      def realize_link
+        link = @options[:url]
+        filename = Digest::SHA256.new.hexdigest(link)
+        measure = Benchmark.measure do
+          File.open(filename, 'w') do |f|
+            open(link) { |rf| f.write(rf.read) }
+          end
+        end
+        puts "Realizing web download from \"#{link}\" took #{measure.real}"
+        filename
+      end
+
+      def realize_s3(params)
+        s3_client = params['aws_client'] && params['aws_client']['s3_client']
+        fail 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:buckets)
+        bucket_name = @options[:bucket]
+        key = @options[:key]
+        fail "Key \"bucket\" is missing in S3 datasource" if bucket_name.blank?
+        fail "Key \"key\" is missing in S3 datasource" if key.blank?
+        puts "Realizing download from S3. Bucket #{bucket_name}, object with key #{key}."
+        filename = Digest::SHA256.new.hexdigest(@options.to_json)
+        bucket = s3_client.buckets[bucket_name]
+        obj = bucket.objects[key]
+        File.open(filename, 'wb') do |file|
+          obj.read do |chunk|
+            file.write(chunk)
+          end
+        end
+        puts 'Done downloading file.'
+        filename
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gooddata-bricks
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Tomas Svarovsky
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-27 00:00:00.000000000 Z
+date: 2016-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -140,6 +140,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: gooddata_datawarehouse
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
@@ -176,6 +190,7 @@ files:
 - lib/gooddata/bricks/middleware/bench_middleware.rb
 - lib/gooddata/bricks/middleware/bulk_salesforce_middleware.rb
 - lib/gooddata/bricks/middleware/decode_params_middleware.rb
+- lib/gooddata/bricks/middleware/dwh_middleware.rb
 - lib/gooddata/bricks/middleware/fs_download_middleware.rb
 - lib/gooddata/bricks/middleware/fs_upload_middleware.rb
 - lib/gooddata/bricks/middleware/gooddata_middleware.rb
@@ -188,6 +203,7 @@ files:
 - lib/gooddata/bricks/pipeline.rb
 - lib/gooddata/bricks/utils.rb
 - lib/gooddata/bricks/version.rb
+- lib/gooddata/helpers/data_helper.rb
 homepage: https://github.com/gooddata/gooddata-bricks
 licenses:
 - BSD