gooddata 0.6.11 → 0.6.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 599d5759ccffd91b5e59aa9fde03eccee81c4138
-  data.tar.gz: 95ec2b6b381f940b686fe29c201c99ec1b1300c9
+  metadata.gz: b9ebfa575ef6b28f90189e3429f1b4563daa44ca
+  data.tar.gz: 1ca90d7f9414423d00baf8dd87634e8e356dd61e
 SHA512:
-  metadata.gz: b82f86ca736a3dbb5d6402c1085a3062c6d3213ff5f2f782f115496408d97d230779870be2923fcb5aa472615c539d65b08c52b022628b06467d2e6ad15816aa
-  data.tar.gz: 018ad7f05291a4f5ce238254d3f24ab3680d2dc0ab4a374990004db785bd1e9b15534cf553d88958ead9eb5bc5b82c8b94507cdc876955144d155283bc016bb7
+  metadata.gz: e1f73a4b8df1f90db97fd13752e336f0c59e05ebf1713471ea6c48f01fddc7702f0f53f4822e74604c3ac70074383058d0a5b91a8e8cdeb24ed0c8e13eb2e197
+  data.tar.gz: 332429c9389f81b6475c31e495a44f93ba9154f7c4886770578a71eb29c82a4140dae49263be12ab6e5cfea6745ecd17d849ac84e97a8301b66ad30f0af299fb

data/.gitignore

@@ -27,3 +27,9 @@ Gemfile.lock
 
 # YARD
 .yardoc/
+
+# Local scripts
+.scripts/
+
+# Temp files
+tmp/

data/.travis.yml

@@ -1,5 +1,9 @@
 language: ruby
 
+# cache: bundler
+
+sudo: false
+
 os:
   - linux
   - osx
@@ -18,6 +22,7 @@ rvm:
   - jruby-19mode
   # - rbx-2.2.9
   - 2.1
+  - 2.2
 
 before_install:
   - gem update --system

data/CHANGELOG.md

@@ -1,12 +1,45 @@
 # GoodData Ruby SDK Changelog
 
+## 0.6.12 (in progress)
+
+- Ability to create a Data Warehouse (ADS) 
+- Retry all requests 3 times when SystemCallError, RestClient::InternalServerError or RestClient::RequestTimeout
+- Automatic 429/TooManyRequests Handler
+- When creating user login and email can be different now
+- Automatic client disconnect at_exit of ruby script
+- When creating user login and email can be different now
+- Fixed Domain#add_user (GH issue #354)
+- Support for GoodData.connect ENV['GD_GEM_USER'], ENV['GD_GEM_PASSWORD']
+- Added Schedule#execute(:wait => true|false) option
+- Merge GoodData::Rest::Connection and GoodData::Rest::Connection::RestClientConnection
+- Unified expection handler for REST API and WebDav Access (using GoodData::Rest::Connection.retryable)
+- GoodData#stats_on, GoodData#stats_off, GoodData::Rest::Client#stats_on, GoodData#stats_off
+- GoodData::Mixin::MdObjectQuery#using now accepts :full => true|false option
+- GoodData::MdObject#[] automatically returns proper type (ie. GoodData::Report)
+- Improved user management
+- Added simple GoodData::Dimension
+
 ## 0.6.11
+
 - Ability to download deployed process
 - Added locking objects capabilities
 - Added removing color mapping form a report definition
 - Report defintions are deleted along with a report
+- Report definitions are deleted along with a report
+- Improved process deployment and schedules
+- Parameters in processes and schedules are now able to take complex parameters
+- #create_metric is significantly faster
+- Pretty_expression for metric should not fail on missing data
+- Extended notation can be switched off when using create_metric
+- Implemented retry on connection related issues
+- All executions should use latest resource version
+- Uploading files to webdav should use streaming and be more memory efficient
+- Ability to pass absolute path to file upload  
+- Allowing special chars in uploaded file
+- GooddataMiddleware doesn't require username+password, when it has SST  
 
 ## 0.6.10
+
 - Fixed client default missing in ProjectMetadata
 - Listing schedules on processes is working
 - Scrubing params in logs is back
@@ -15,8 +48,8 @@
 - Schedule can be enabled/disabled
 - Added pselect helper function
 
-
 ## 0.6.9
+
 - Fixing issues with creating models.
 - Adding couple more helpers for report/metric computation
 - Rewriting several full_* specs to use the new syntax

data/CLI.md

@@ -259,7 +259,7 @@ Shows basic info about a project
 tomaskorcak@kx-mac:~/$ gooddata -p tk6192gsnav58crp6o1ahsmtuniq8khb project show
 {"content"=>
   {"cluster"=>"",
-   "authorizationToken"=>"INTNA000000SRVS",
+   "authorizationToken"=>"IOUYYUY8786",
    "guidedNavigation"=>"1",
    "isPublic"=>"0",
    "driver"=>"Pg",

data/authors.sh

@@ -0,0 +1,4 @@
+#! /usr/bin/env bash
+
+git shortlog -s -n
+

data/lib/gooddata.rb

@@ -1,7 +1,7 @@
 # encoding: UTF-8
 
 require 'pmap'
-$pmap_default_thread_count = 20 # rubocop:disable GlobalVars
+$pmap_default_thread_count = 100 # rubocop:disable GlobalVars
 
 # GoodData Module
 module GoodData

data/lib/gooddata/cli/commands/api_cmd.rb

@@ -9,7 +9,6 @@ GoodData::CLI.module_eval do
   desc 'Some basic API stuff directly from CLI'
   arg_name 'info|test|get|delete'
   command :api do |c|
-
     c.desc 'Info about the API version etc'
     c.command :info do |info|
       info.action do |global_options, options, _args|
@@ -27,6 +26,5 @@ GoodData::CLI.module_eval do
         pp GoodData::Command::Api.get(args[0])
       end
     end
-
   end
 end

data/lib/gooddata/cli/commands/auth_cmd.rb

@@ -4,10 +4,8 @@ require_relative '../shared'
 require_relative '../../commands/auth'
 
 GoodData::CLI.module_eval do
-
   desc 'Work with your locally stored credentials'
   command :auth do |c|
-
     c.desc 'Store your credentials to ~/.gooddata so client does not have to ask you every single time'
     c.command :store do |store|
       store.action do |_global_options, _options, _args|
@@ -15,5 +13,4 @@ GoodData::CLI.module_eval do
       end
     end
   end
-
 end

data/lib/gooddata/cli/commands/console_cmd.rb

@@ -7,7 +7,6 @@ require_relative '../shared'
 GoodData::CLI.module_eval do
   desc 'Interactive session with gooddata sdk loaded'
   command :console do |c|
-
     c.action do |global_options, _options, _args|
       username = global_options[:username]
       fail ArgumentError, 'No username specified' if username.nil? || username.empty?
@@ -28,7 +27,7 @@ GoodData::CLI.module_eval do
         puts "Use 'exit' to quit the live session. Use 'q' to jump out of displaying a large output."
         binding.pry(:quiet => true,
                     :prompt => [proc do |_target_self, _nest_level, _pry|
-                      'sdk_live_sesion: '
+                      'sdk_live_session: '
                     end])
       end
       client.disconnect

data/lib/gooddata/cli/commands/domain_cmd.rb

@@ -4,10 +4,8 @@ require_relative '../shared'
 require_relative '../../commands/domain'
 
 GoodData::CLI.module_eval do
-
   desc 'Manage domain'
   command :domain do |c|
-
     c.desc 'Add user to domain'
     c.command :add_user do |add_user|
       add_user.action do |global_options, options, args|

data/lib/gooddata/cli/commands/process_cmd.rb

@@ -16,11 +16,9 @@ def read_params_file(filename)
 end
 
 GoodData::CLI.module_eval do
-
   desc 'Work with deployed processes'
   arg_name 'Describe arguments to list here'
   command :process do |c|
-
     c.desc 'Use when you need to redeploy a specific process'
     c.default_value nil
     c.flag :process_id

data/lib/gooddata/cli/commands/project_cmd.rb

@@ -7,7 +7,6 @@ require_relative '../shared'
 require_relative '../../commands/project'
 
 GoodData::CLI.module_eval do
-
   desc 'Manage your project'
   arg_name 'project_command'
   command :project do |c|
@@ -140,7 +139,6 @@ GoodData::CLI.module_eval do
     c.desc 'If you are in a gooddata project blueprint it will apply the changes. If you do not provide a project id it will build it from scratch and create a project for you.'
     c.command :update do |show|
       show.action do |global_options, options, _args|
-
         opts = options.merge(global_options)
         GoodData.connect(opts)
         spec, project_id = GoodData::Command::Project.get_spec_and_project_id('.')

data/lib/gooddata/cli/commands/projects_cmd.rb

@@ -4,10 +4,8 @@ require_relative '../shared'
 require_relative '../../commands/projects'
 
 GoodData::CLI.module_eval do
-
   desc 'Manage your projects'
   command :projects do |c|
-
     c.desc "Lists user's projects"
     c.command :list do |list|
       list.action do |global_options, options, _args|

data/lib/gooddata/cli/commands/run_ruby_cmd.rb

@@ -15,11 +15,9 @@ def load_undot(filename)
 end
 
 GoodData::CLI.module_eval do
-
   desc 'Run ruby bricks either locally or remotely deployed on our server. Currently private alpha.'
   # arg_name 'show'
   command :run_ruby do |c|
-
     c.desc 'Directory of the ruby brick'
     c.default_value nil
     c.flag [:d, :dir]
@@ -30,7 +28,7 @@ GoodData::CLI.module_eval do
 
     c.desc 'Params file path. Inside should be hash of key values. These params override any defaults given in bricks.'
     c.default_value nil
-    c.flag [:params]
+    c.flag [:params, :paramfile]
 
     c.desc 'Remote system credentials file path. Inside should be hash of key values.'
     c.default_value nil
@@ -55,6 +53,7 @@ GoodData::CLI.module_eval do
                                   else
                                     { 'config' => {} }
                                   end
+
       # if there are some GDC_* params in config, put them on the level above
       gdc_params = options[:expanded_params]['config'].select { |k, _| k =~ /GDC_.*/ }
       options[:expanded_params].merge!(gdc_params)

data/lib/gooddata/cli/commands/scaffold_cmd.rb

@@ -6,11 +6,9 @@ require_relative '../shared'
 require_relative '../../commands/scaffold'
 
 GoodData::CLI.module_eval do
-
   desc 'Scaffold things'
   arg_name 'show'
   command :scaffold do |c|
-
     c.desc 'Scaffold a gooddata project blueprint'
     c.command :project do |project|
       project.action do |_global_options, _options, args|
@@ -30,5 +28,4 @@ GoodData::CLI.module_eval do
       end
     end
   end
-
 end

data/lib/gooddata/cli/commands/user_cmd.rb

@@ -6,7 +6,6 @@ require_relative '../../commands/role'
 require_relative '../../commands/user'
 
 GoodData::CLI.module_eval do
-
   desc 'User management'
   command :user do |c|
     c.desc 'Show your profile'
@@ -18,5 +17,4 @@ GoodData::CLI.module_eval do
       end
     end
   end
-
 end

data/lib/gooddata/cli/shared.rb

@@ -5,11 +5,11 @@ require 'gli'
 require_relative '../version'
 require_relative '../core/core'
 require_relative '../extensions/extensions'
+require_relative '../exceptions/exceptions'
 
 include GLI::App
 
 GoodData::CLI.module_eval do
-
   program_desc 'GoodData Ruby gem - a wrapper over GoodData API and several useful abstractions to make your everyday usage of GoodData easier.'
 
   version GoodData::VERSION
@@ -51,5 +51,4 @@ GoodData::CLI.module_eval do
   desc 'Http logger on stdout'
   arg_name 'logger'
   switch [:l, :logger]
-
 end

data/lib/gooddata/commands/datawarehouse.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+
+module GoodData
+  module Command
+    # Also known as ADS and DSS
+    class DataWarehouse
+      class << self
+        # Create new project based on options supplied
+        def create(options = { client: GoodData.connection })
+          title = options[:title]
+          description = options[:summary] || options[:description]
+          token = options[:token] || options[:auth_token]
+          client = options[:client]
+          GoodData::DataWarehouse.create(
+                                    :title => title,
+                                    :description => description,
+                                    :auth_token => token,
+                                    :client => client
+                                  )
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/commands/process.rb

@@ -52,7 +52,6 @@ module GoodData
           verbose = options[:v]
           dir = Pathname(dir)
           name = options[:name] || "Temporary deploy[#{dir}][#{options[:project_name]}]"
-
           GoodData::Process.with_deploy(dir, options.merge(:name => name, :project_id => ProjectHelper::PROJECT_ID)) do |process|
             puts HighLine.color('Executing', HighLine::BOLD) if verbose
             process.execute(executable, options)

data/lib/gooddata/commands/project.rb

@@ -126,7 +126,7 @@ module GoodData
                 puts "Use 'exit' to quit the live session. Use 'q' to jump out of displaying a large output."
                 binding.pry(:quiet => true,
                             :prompt => [proc do |_target_self, _nest_level, _pry|
-                              'project_live_sesion: '
+                              'project_live_session: '
                             end])
               end
             rescue GoodData::ProjectNotFound

data/lib/gooddata/commands/scaffold.rb

@@ -17,7 +17,6 @@ module GoodData
 
           FileUtils.mkdir(name)
           FileUtils.cd(name) do
-
             FileUtils.mkdir('model')
             FileUtils.cd('model') do
               input = File.read(TEMPLATES_PATH + 'project/model/model.rb.erb')

data/lib/gooddata/core/connection.rb

@@ -0,0 +1,376 @@
+# encoding: UTF-8
+
+require 'multi_json'
+require 'rest-client'
+
+require_relative '../version'
+
+module GoodData
+  # # GoodData HTTP wrapper
+  #
+  # Provides a convenient HTTP wrapper for talking with the GoodData API.
+  #
+  # Remember that the connection is shared amongst the entire application.
+  # Therefore you can't be logged in to more than _one_ GoodData account.
+  # per session. Simultaneous connections to multiple GoodData accounts is not
+  # supported at this time.
+  #
+  # The GoodData API is a RESTful API that communicates using JSON. This wrapper
+  # makes sure that the session is stored between requests and that the JSON is
+  # parsed both when sending and receiving.
+  #
+  # ## Usage
+  #
+  # Before a connection can be made to the GoodData API, you have to supply the user credentials like this:
+  #
+  #   Connection.new(username, password)
+  #
+  # To send a HTTP request use either the get, post or delete methods documented below.
+  #
+  class Connection
+    DEFAULT_URL = 'https://secure.gooddata.com'
+    LOGIN_PATH = '/gdc/account/login'
+    TOKEN_PATH = '/gdc/account/token'
+
+    attr_reader(:auth_token, :url)
+    attr_accessor :status, :options
+
+    # Options:
+    # * :tries - Number of retries to perform. Defaults to 1.
+    # * :on - The Exception on which a retry will be performed. Defaults to Exception, which retries on any Exception.
+    #
+    # ### Example
+    #
+    #     retryable(:tries => 1, :on => OpenURI::HTTPError) do
+    #       # your code here
+    #     end
+    #
+    def retryable(options = {})
+      opts = { :tries => 1, :on => Exception }.merge(options)
+
+      retry_exception, retries = opts[:on], opts[:tries]
+
+      begin
+        return yield
+      rescue retry_exception
+        retry if (retries -= 1) > 0
+      end
+
+      yield
+    end
+
+    # Set the GoodData account credentials.
+    #
+    # This have to be performed before any calls to the API.
+    #
+    # @param username The GoodData account username
+    # @param password The GoodData account password
+    #
+    def initialize(username, password, options = {})
+      @status = :not_connected
+      @username = username
+      @password = password
+      @url = options[:server] || DEFAULT_URL
+      @auth_token = options[:gdc_temporary_token]
+      @options = options
+
+      @headers = options[:headers] || {}
+
+      default_headers = {
+        :content_type => :json,
+        :accept => [:json, :zip],
+        :user_agent => GoodData.gem_version_string
+      }
+      default_headers.merge! @headers
+
+      @server = RestClient::Resource.new @url,
+                                         :timeout => @options[:timeout],
+                                         :headers => default_headers
+
+      @server = create_server_connection(@url, @options)
+    end
+
+    # Returns the user JSON object of the currently logged in GoodData user account.
+    def user
+      ensure_connection
+      @user
+    end
+
+    # Performs a HTTP GET request.
+    #
+    # Retuns the JSON response formatted as a Hash object.
+    #
+    # @param path The HTTP path on the GoodData server (must be prefixed with a forward slash)
+    #
+    # ### Examples
+    #
+    #     Connection.new(username, password).get '/gdc/projects'
+    #
+    def get(path, options = {})
+      GoodData.logger.debug "GET #{@server}#{path}"
+      ensure_connection
+      b = proc { @server[path].get cookies }
+      process_response(options, &b)
+    end
+
+    # Performs a HTTP POST request.
+    #
+    # Retuns the JSON response formatted as a Hash object.
+    #
+    # @param path The HTTP path on the GoodData server (must be prefixed with a forward slash)
+    # @param data The payload data in the format of a Hash object
+    #
+    # ### Examples
+    #
+    #     Connection.new(username, password).post '/gdc/projects', { ... }
+    #
+    def post(path, data, options = {})
+      GoodData.logger.debug("POST #{@server}#{path}, payload: #{scrub_params(data, [:password, :login, :authorizationToken])}")
+      ensure_connection
+      payload = data.is_a?(Hash) ? data.to_json : data
+      b = proc { @server[path].post payload, cookies }
+      process_response(options, &b)
+    end
+
+    # Performs a HTTP PUT request.
+    #
+    # Retuns the JSON response formatted as a Hash object.
+    #
+    # @param path The HTTP path on the GoodData server (must be prefixed with a forward slash)
+    # @param data The payload data in the format of a Hash object
+    #
+    # ### Examples
+    #
+    #     Connection.new(username, password).put '/gdc/projects', { ... }
+    #
+    def put(path, data, options = {})
+      payload = data.is_a?(Hash) ? data.to_json : data
+      GoodData.logger.debug "PUT #{@server}#{path}, payload: #{payload}"
+      ensure_connection
+      b = proc { @server[path].put payload, cookies }
+      process_response(options, &b)
+    end
+
+    # Performs a HTTP DELETE request.
+    #
+    # Retuns the JSON response formatted as a Hash object.
+    #
+    # @param path The HTTP path on the GoodData server (must be prefixed with a forward slash)
+    #
+    # ### Examples
+    #
+    #     Connection.new(username, password).delete '/gdc/project/1'
+    #
+    def delete(path, options = {})
+      GoodData.logger.debug "DELETE #{@server}#{path}"
+      ensure_connection
+      b = proc { @server[path].delete cookies }
+      process_response(options, &b)
+    end
+
+    # Get the cookies associated with the current connection.
+    def cookies
+      @cookies ||= { :cookies => {} }
+    end
+
+    # Set the cookies used when communicating with the GoodData API.
+    def merge_cookies!(cookies)
+      self.cookies
+      @cookies[:cookies].merge! cookies
+    end
+
+    # Returns true if a connection have been established to the GoodData API
+    # and the login was successful.
+    def logged_in?
+      @status == :logged_in
+    end
+
+    def url=(url = nil)
+      @url = url || DEFAULT_URL
+      @server = create_server_connection(@url, @options)
+    end
+
+    # The connection will automatically be established once it's needed, which it
+    # usually is when either the user, get, post or delete method is called. If you
+    # want to force a connection (or a re-connect) you can use this method.
+    def connect!
+      connect
+    end
+
+    # Uploads a file to GoodData server
+    # /uploads/ resources are special in that they use a different
+    # host and a basic authentication.
+    def upload(file, options = {})
+      ensure_connection
+
+      dir = options[:directory] || ''
+      staging_uri = options[:staging_url].to_s
+      url = dir.empty? ? staging_uri : URI.join(staging_uri, "#{dir}/").to_s
+
+      # Make a directory, if needed
+      unless dir.empty?
+        method = :get
+        GoodData.logger.debug "#{method}: #{url}"
+        begin
+          # first check if it does exits
+          RestClient::Request.execute({
+            :method => method,
+            :url => url,
+            :timeout => @options[:timeout],
+            :headers => {
+              :user_agent => GoodData.gem_version_string
+            }
+          }.merge(cookies))
+        rescue RestClient::Exception => e
+          if e.http_code == 404
+            method = :mkcol
+            GoodData.logger.debug "#{method}: #{url}"
+            RestClient::Request.execute({
+              :method => method,
+              :url => url,
+              :timeout => @options[:timeout],
+              :headers => {
+                :user_agent => GoodData.gem_version_string
+              }
+            }.merge(cookies))
+          end
+        end
+      end
+
+      payload = options[:stream] ? 'file' : File.read(file)
+      filename = options[:filename] || options[:stream] ? 'randome-filename.txt' : File.basename(file)
+
+      # Upload the file
+      # puts "uploading the file #{URI.join(url, filename).to_s}"
+      req = RestClient::Request.new(
+        :method => :put,
+        :url => URI.join(url, filename).to_s,
+        :timeout => @options[:timeout],
+        :headers => {
+          :user_agent => GoodData.gem_version_string
+        },
+        :payload => payload,
+        :raw_response => true,
+        :user => @username,
+        :password => @password
+      )
+      # .merge(cookies))
+      req.execute
+      true
+    end
+
+    def download(what, where, options = {})
+      staging_uri = options[:staging_url].to_s
+      url = staging_uri + what
+      req = RestClient::Request.new({
+        :method => 'GET',
+        :url => url,
+        :user => @username,
+        :password => @password
+      }.merge(cookies))
+
+      if where.is_a?(String)
+        File.open(where, 'w') do |f|
+          req.execute do |chunk, _, _|
+            f.write chunk
+          end
+        end
+      else
+        # Assume it is a IO stream
+        req.execute do |chunk, _, _|
+          where.write chunk
+        end
+      end
+    end
+
+    def connected?
+      @status == :logged_in
+    end
+
+    def disconnect
+      return if !connected? && !GoodData.connection.user['state']
+      GoodData.delete(GoodData.connection.user['state'])
+      @status = :not_connected
+    end
+
+    private
+
+    def create_server_connection(url, options)
+      RestClient::Resource.new url,
+                               :timeout => options[:timeout],
+                               :headers => {
+                                 :content_type => :json,
+                                 :accept => [:json, :zip],
+                                 :user_agent => GoodData.gem_version_string
+                               }
+    end
+
+    def ensure_connection
+      connect if @status == :not_connected
+    end
+
+    def connect
+      GoodData.logger.info 'Connecting to GoodData...'
+      @status = :connecting
+      authenticate
+    end
+
+    def authenticate
+      credentials = {
+        'postUserLogin' => {
+          'login' => @username,
+          'password' => @password,
+          'remember' => 1
+        }
+      }
+      GoodData.logger.debug 'Logging in...'
+      @user = post(LOGIN_PATH, credentials, :dont_reauth => true)['userLogin']
+      refresh_token :dont_reauth => true # avoid infinite loop if refresh_token fails with 401
+
+      @status = :logged_in
+    end
+
+    def process_response(options = {}, &block)
+      begin
+        response = block.call
+      rescue RestClient::Unauthorized
+        raise $ERROR_INFO if options[:dont_reauth]
+        refresh_token
+        response = block.call
+      end
+      merge_cookies! response.cookies
+      content_type = response.headers[:content_type]
+      return response if options[:process] == false
+
+      if content_type == 'application/json' || content_type == 'application/json;charset=UTF-8'
+        result = response.to_str == '""' ? {} : MultiJson.load(response.to_str)
+        GoodData.logger.debug "Response: #{result.inspect}"
+      elsif content_type == 'application/zip'
+        result = response
+        GoodData.logger.debug 'Response: a zipped stream'
+      elsif response.headers[:content_length].to_s == '0'
+        result = nil
+        GoodData.logger.debug 'Response: Empty response possibly 204'
+      elsif response.code == 204
+        result = nil
+        GoodData.logger.debug 'Response: 204 no content'
+      else
+        fail "Unsupported response content type '%s':\n%s" % [content_type, response.to_str[0..127]]
+      end
+      result
+    rescue RestClient::Exception => e
+      GoodData.logger.debug "Response: #{e.response}"
+      raise $ERROR_INFO
+    end
+
+    def refresh_token(options = {})
+      GoodData.logger.debug 'Getting authentication token...'
+      begin
+        get TOKEN_PATH, :dont_reauth => true # avoid infinite loop GET fails with 401
+      rescue RestClient::Unauthorized
+        raise $ERROR_INFO if options[:dont_reauth]
+        authenticate
+      end
+    end
+  end
+end