gooddata 0.6.15 → 0.6.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2be96271a19b80a9bbb9658234c19fafbeadf935
-  data.tar.gz: 11a5fd29bede6c2258a0954d2e5b0fc6fd8d71ec
+  metadata.gz: 40814e8c9d7deee9ac0eca814a3ac3c635f1ede3
+  data.tar.gz: fe14bb4f38e410de2e7abc9624e5f3aba6c39e6f
 SHA512:
-  metadata.gz: 8590dd4828f12d0bb49beadaafc6b32080678c5df6d38e8fc710a77f2ae54983e6cd5cdeee04c1fc835b91774240e8c2da68486f9c649e101b2131ababca5301
-  data.tar.gz: de42c006bb3bed502feab54273fa3263f4475f081617de84d5b6eb8746941e1e55d5cf814fab09e47801a9691561b192a5791f6bdaa13e53c1b6a6cdcbb44e64
+  metadata.gz: f4128f12fab224bc8a5814fe0dcee35c385c2652defe34afb331562a65332b7f4fdd9199b7bb4c653eebdff39d6dd21950482e10fb1b51c5e81664072c490035
+  data.tar.gz: 731f9c3dab1e859777931f44362e4e50c2bc79fa562a4172ad2d53c658d9b3f309ff2000fb36a9090bdecd09be5240d76d3344316d60b281803bcc3d4e90373e

data/lib/gooddata/bricks/middleware/fs_download_middleware.rb

@@ -0,0 +1,42 @@
+# encoding: UTF-8
+require_relative 'base_middleware'
+
+module GoodData
+  module Bricks
+    class FsProjectDownloadMiddleware < Bricks::Middleware
+      def call(params)
+        (params['gdc_files_to_download'] || []).each do |source|
+          case source[:type].to_s
+          when 'ads'
+            CSV.open(source[:to], 'w') do |csv|
+              header_written = false
+              header = nil
+              dwh = params['ads_client']
+              dwh.execute_select(source[:query]) do |row|
+                unless header_written
+                  header_written = true
+                  header = row.keys
+                  csv << header
+                end
+                csv << row.values_at(*header)
+              end
+            end
+          when 'staging'
+            webdav_uri = GoodData.project_webdav_path
+            dav = Net::DAV.new(webdav_uri, :curl => false)
+            dav.verify_server = false
+            dav.credentials(params['GDC_USERNAME'], params['GDC_PASSWORD'])
+            dav.find(path, recursive: true, suppress_errors: true) do |item|
+              puts 'Checking: ' + item.url.to_s
+              name = (item.uri - webdav_uri).to_s
+              File.open(name, 'w') do |f|
+                f << item.content
+              end
+            end
+          end
+        end
+        @app.call(params)
+      end
+    end
+  end
+end

data/lib/gooddata/bricks/middleware/fs_upload_middleware.rb

@@ -1,9 +1,4 @@
 # encoding: UTF-8
-
-require 'uri'
-require 'net/http'
-require 'pathname'
-
 require_relative 'base_middleware'
 
 module GoodData
@@ -15,20 +10,21 @@ module GoodData
       end
 
       def call(params)
-        returning(@app.call(params)) do |_result|
+        returning(@app.call(params)) do |_|
           destination = @destination
           (params['gdc_files_to_upload'] || []).each do |f|
-            case destination.to_s
-            when 'staging'
-              GoodData.upload_to_project_webdav(f[:path], directory: f[:webdav_directory])
-              puts "Uploaded local file \"#{f[:path]}\" to webdav."
+            path = f[:path]
+            case destination.to_sym
+            when :staging
+              GoodData.client.get '/gdc/account/token', :dont_reauth => true
+              url = GoodData.project_webdav_path
+              GoodData.upload_to_project_webdav(path)
+              puts "Uploaded local file \"#{path}\" to url \"#{url + path}\""
             end
           end
         end
       end
     end
-
-    # Alias to make it backwards compatible
     FsUploadMiddleware = FsProjectUploadMiddleware
   end
 end

data/lib/gooddata/bricks/middleware/gooddata_middleware.rb

@@ -1,8 +1,4 @@
 # encoding: UTF-8
-
-# TODO: REmove this
-# require 'gooddata'
-
 require_relative 'base_middleware'
 
 module GoodData
@@ -11,25 +7,26 @@ module GoodData
       def call(params)
         logger = params['GDC_LOGGER']
         token_name = 'GDC_SST'
-        protocol_name = 'GDC_PROTOCOL'
-        server_name = 'GDC_HOSTNAME'
+        protocol_name = 'CLIENT_GDC_PROTOCOL'
+        server_name = 'CLIENT_GDC_HOSTNAME'
         project_id = params['GDC_PROJECT_ID']
 
-        fail 'SST (SuperSecureToken) not present in params' if params[token_name].nil?
-
-        server = if !params[protocol_name].empty? && !params[server_name].empty?
+        server = if params[protocol_name] && params[server_name]
                    "#{params[protocol_name]}://#{params[server_name]}"
                  end
 
         client = if params['GDC_USERNAME'].nil? || params['GDC_PASSWORD'].nil?
-                   GoodData.connect(sst_token: params['GDC_SST'], server: server)
+                   puts "Connecting with SST to server #{server}"
+                   fail 'SST (SuperSecureToken) not present in params' if params[token_name].nil?
+                   GoodData.connect(sst_token: params[token_name], server: server)
                  else
+                   puts "Connecting as #{params['GDC_USERNAME']} to server #{server}"
                    GoodData.connect(params['GDC_USERNAME'], params['GDC_PASSWORD'], server: server)
                  end
+        project = client.projects(project_id)
+        GoodData.project = project
         GoodData.logger = logger
-        GoodData.with_project(project_id, :client => client) do |_p|
-          @app.call(params)
-        end
+        @app.call(params.merge('GDC_GD_CLIENT' => client, 'gdc_project' => project))
       end
     end
   end

data/lib/gooddata/bricks/middleware/logger_middleware.rb

@@ -8,8 +8,13 @@ module GoodData
   module Bricks
     class LoggerMiddleware < Bricks::Middleware
       def call(params)
-        logger = params['GDC_LOGGER'] = params[:GDC_LOGGER_FILE].nil? ? Logger.new(STDOUT) : Logger.new(params[:GDC_LOGGER_FILE])
-        logger.info('Pipeline starts')
+        logger = nil
+        if params['GDC_LOGGING_OFF']
+          logger = NilLogger.new
+        else
+          logger = params['GDC_LOGGER'] = params[:GDC_LOGGER_FILE].nil? ? Logger.new(STDOUT) : Logger.new(params[:GDC_LOGGER_FILE])
+          logger.info('Pipeline starts')
+        end
         returning(@app.call(params)) do |_result|
           logger.info('Pipeline ending')
         end

data/lib/gooddata/core/rest.rb

@@ -77,39 +77,38 @@ module GoodData
 
     # Get WebDav directory for project data
     # @return [String]
-    def get_project_webdav_path(file, options = { :project => GoodData.project })
+    def project_webdav_path(options = { :project => GoodData.project })
       options = merge_options(options)
       project = options[:project]
-      project.get_project_webdav_path(file)
+      project.project_webdav_path
     end
 
     # Upload to project directory
     def upload_to_project_webdav(file, options = { :project => GoodData.project })
       options = merge_options(options)
-      webdav_filename = File.basename(file)
-      url = get_project_webdav_path(webdav_filename, options)
+      url = project_webdav_path(options)
       connection.upload(file, options.merge(:staging_url => url))
     end
 
     # Download from project directory
     def download_from_project_webdav(file, where, options = { :project => GoodData.project })
       options = merge_options(options)
-      url = get_project_webdav_path(file, options)
+      url = project_webdav_path(options)
       connection.download(file, where, options.merge(:staging_url => url))
     end
 
     # Get WebDav directory for user data
     # @return [String]
-    def get_user_webdav_path(file, options = { :project => GoodData.project })
+    def user_webdav_path(options = { :project => GoodData.project })
       options = merge_options(options)
       project = options[:project]
-      project.get_user_webdav_path(file)
+      project.user_webdav_path
     end
 
     # Download from user directory
     def download_from_user_webdav(file, where, options = { :project => GoodData.project })
       options = merge_options(options)
-      url = get_user_webdav_path(file, options)
+      url = user_webdav_path(options)
       connection.download(file, where, options.merge(:staging_url => url))
     end
 

data/lib/gooddata/helpers/data_helper.rb

@@ -0,0 +1,81 @@
+# encoding: UTF-8
+
+require 'csv'
+require 'digest'
+require 'open-uri'
+
+module GoodData
+  module Helpers
+    class DataSource
+      attr_reader :realized
+
+      def initialize(opts = {})
+        opts = opts.is_a?(String) ? { type: :staging, path: opts } : opts
+        opts = opts.symbolize_keys
+        @source = opts[:type]
+        @options = opts
+        @realized = false
+      end
+
+      def realize_query(params)
+        query = @options[:query]
+        dwh = params['ads_client']
+        fail "Data Source needs a client to ads to be able to query the storage but 'ads_client' is empty." unless dwh
+        filename = Digest::SHA256.new.hexdigest(query)
+        measure = Benchmark.measure do
+          CSV.open(filename, 'w') do |csv|
+            header_written = false
+            header = nil
+
+            dwh.execute_select(query) do |row|
+              unless header_written
+                header_written = true
+                header = row.keys
+                csv << header
+              end
+              csv << row.values_at(*header)
+            end
+          end
+        end
+        puts "Realizing SQL query \"#{query}\" took #{measure.real}"
+        filename
+      end
+
+      def realize_staging(params)
+        path = @options[:path]
+        url = URI.parse(path)
+        filename = Digest::SHA256.new.hexdigest(path)
+        if url.relative?
+          params['gdc_project'].download_file(path, filename)
+        else
+          params['GDC_GD_CLIENT'].download_file(path, filename)
+        end
+        filename
+      end
+
+      def realize_link
+        link = @options[:url]
+        filename = Digest::SHA256.new.hexdigest(link)
+        measure = Benchmark.measure do
+          File.open(filename, 'w') do |f|
+            open(link) { |rf| f.write(rf.read) }
+          end
+        end
+        puts "Realizing web download from \"#{link}\" took #{measure.real}"
+        filename
+      end
+
+      def realize(params = {})
+        @realized = true
+        case @source.to_s
+        when 'ads'
+          realize_query(params)
+        when 'staging'
+          realize_staging(params)
+        when 'web'
+          realize_link
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/helpers/global_helpers.rb

@@ -120,7 +120,7 @@ module GoodData
             nested[k] = v
           end
         end
-        key = hidden ? ENCODED_PARAMS_KEY : ENCODED_HIDDEN_PARAMS_KEY
+        key = hidden ? ENCODED_HIDDEN_PARAMS_KEY : ENCODED_PARAMS_KEY
         res[key] = nested.to_json unless nested.empty?
         res
       end

data/lib/gooddata/models/domain.rb

@@ -10,8 +10,6 @@ module GoodData
   class Domain < GoodData::Rest::Object
     attr_reader :name
 
-    USERS_OPTIONS = { :offset => 0, :limit => 10_000 }
-
     class << self
       # Looks for domain
       #
@@ -182,21 +180,22 @@ module GoodData
       # @option opts [Number] :offset The subject
       # @option opts [Number] :limit From address
       # TODO: Review opts[:limit] functionality
-      def users(domain, opts = USERS_OPTIONS.merge(:client => GoodData.connection))
+      def users(domain, opts = {})
         result = []
-
-        offset = 0 || opts[:offset]
-        uri = "/gdc/account/domains/#{domain}/users?offset=#{offset}&limit=#{opts[:limit]}"
+        page_limit = opts[:page_limit] || 1000
+        limit = opts[:limit] || Float::INFINITY
+        offset = opts[:offset]
+        uri = "/gdc/account/domains/#{domain}/users?offset=#{offset}&limit=#{page_limit}"
         loop do
-          break unless uri
           tmp = client(opts).get(uri)
           tmp['accountSettings']['items'].each do |account|
             result << client(opts).create(GoodData::Profile, account)
           end
-          break if opts[:limit] && result.length >= opts[:limit]
+          break if result.length >= limit
+
           uri = tmp['accountSettings']['paging']['next']
+          break unless uri
         end
-
         result
       end
 
@@ -205,51 +204,26 @@ module GoodData
       # @param [String] default_domain_name Default domain name used when no specified in user
       # @return [Array<GoodData::User>] List of users created
       def create_users(list, default_domain = nil, opts = { :client => GoodData.connection, :project => GoodData.project })
-        client = client(opts)
         default_domain_name = default_domain.respond_to?(:name) ? default_domain.name : default_domain
-        domain_obj = client.domain(default_domain_name)
-        domains = {}
-        list.map do |user|
+        domain = client.domain(default_domain_name)
+
+        domain_users_cache = Hash[domain.users.map { |u| [u.login, u] }]
+        list.pmapcat do |user|
           begin
             user_data = user.to_hash
-            # TODO: Add user here
-            domain_name = user_data[:domain] || default_domain_name
-
-            # Lookup for domain in cache'
-            domain = domains[domain_name]
-
-            # Get domain info from REST, add to cache
-            if domain.nil?
-              domain = {
-                :domain => domain_obj,
-                :users => domain_obj.users
-              }
-
-              domain[:users_map] = Hash[domain[:users].map { |u| [u.login, u] }]
-              domains[domain_name] = domain
-            end
-
-            # Check if user exists in domain
-            domain_user = domain[:users_map][user_data[:login]]
-
-            # Create domain user if needed
+            domain_user = domain_users_cache[user_data[:login]]
             if !domain_user
-              # Add created user to cache
-              domain_user = domain[:domain].add_user(user_data, opts)
-              domain[:users] << domain_user
-              domain[:users_map][domain_user.login] = domain_user
-              { type: :user_added_to_domain, user: domain_user }
+              added_user = domain.add_user(user_data, opts)
+              [{ type: :user_added_to_domain, user: added_user }]
             else
-              # fields = [:firstName, :email]
-              diff = GoodData::Helpers.diff([domain_user.to_hash], [user_data], key: :login)
-              next if diff[:changed].empty?
-
-              domain_user = domain[:domain].update_user(domain_user.to_hash.merge(user_data.compact), opts)
-              domain[:users_map][domain_user.login] = domain_user
-              { type: :user_changed_in_domain, user: domain_user }
+              fields_to_check = opts[:fields_to_check] || user_data.keys
+              diff = GoodData::Helpers.diff([domain_user.to_hash], [user_data], key: :login, fields: fields_to_check)
+              next [] if diff[:changed].empty?
+              updated_user = domain.update_user(domain_user.to_hash.merge(user_data.compact), opts)
+              [{ type: :user_changed_in_domain, user: updated_user }]
             end
           rescue RuntimeError => e
-            { type: :error, reason: e }
+            [{ type: :error, reason: e }]
           end
         end
       end
@@ -358,7 +332,7 @@ module GoodData
     # domain = GoodData::Domain['gooddata-tomas-korcak']
     # pp domain.users
     #
-    def users(opts = USERS_OPTIONS)
+    def users(opts = {})
       GoodData::Domain.users(name, opts.merge(client: client))
     end
 

data/lib/gooddata/models/membership.rb

@@ -50,9 +50,11 @@ module GoodData
               # And following lines are even much more ugly hack
               # 'authentication_modes' => ['sso', 'password']
             },
+            'links' => {},
             'meta' => {}
           }
         }
+        json['user']['links']['self'] = data[:uri] if data[:uri]
         c.create(self, json)
       end
 

data/lib/gooddata/models/metadata/attribute.rb

@@ -123,7 +123,7 @@ module GoodData
     # @param name [String] name used as a basis for regular expression
     # @return [GoodData::Label]
     def label_by_name(name)
-      labels.find { |label| label.title.downcase =~ /#{name}/ || label.identifier.downcase =~ /#{name}/ }
+      labels.find { |label| label.title =~ /#{name}/ || label.identifier =~ /#{name}/ }
     end
   end
 end

data/lib/gooddata/models/metadata/dataset.rb

@@ -35,7 +35,7 @@ module GoodData
     #
     # @return [Array<GoodData::Fact>]
     def facts
-      fact_uris.pmap { |a_uri| project.attributes(a_uri) }
+      fact_uris.pmap { |a_uri| project.facts(a_uri) }
     end
 
     # Gives you list of fact uris on a dataset

data/lib/gooddata/models/process.rb

@@ -167,9 +167,10 @@ module GoodData
           path
         else
           with_zip(opts) do |zipfile|
-            Dir[File.join(path, '**', '**')].reject { |f| files_to_exclude.include?(Pathname(path) + f) }.each do |file|
+            files_to_upload = Dir[File.join(path, '**', '**')].reject { |f| files_to_exclude.include?(Pathname(path) + f) }
+            puts "Uploading #{files_to_upload.count} files."
+            files_to_upload.each do |file|
               file_pathname = Pathname.new(file)
-              puts "Including item #{file_pathname}"
               file_relative_pathname = file_pathname.relative_path_from(Pathname.new(path))
               zipfile.add(file_relative_pathname, file)
             end