gooddata-edge 0.6.27.edge

data/lib/gooddata/helpers/data_helper.rb

@@ -0,0 +1,116 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'csv'
+require 'digest'
+require 'open-uri'
+
+module GoodData
+  module Helpers
+    class DataSource
+      attr_reader :realized
+
+      def initialize(opts = {})
+        opts = opts.is_a?(String) ? { type: :staging, path: opts } : opts
+        opts = GoodData::Helpers.symbolize_keys(opts)
+        @source = opts[:type]
+        @options = opts
+        @realized = false
+      end
+
+      def realize(params = {})
+        @realized = true
+        source = @source && @source.to_s
+        case source
+        when 'ads'
+          realize_query(params)
+        when 'staging'
+          realize_staging(params)
+        when 'web'
+          realize_link
+        when 's3'
+          realize_s3(params)
+        else
+          fail "DataSource does not support type \"#{source}\""
+        end
+      end
+
+      def realized?
+        @realized == true
+      end
+
+      private
+
+      def realize_query(params)
+        query = @options[:query]
+        dwh = params['ads_client']
+        fail "Data Source needs a client to ads to be able to query the storage but 'ads_client' is empty." unless dwh
+        filename = Digest::SHA256.new.hexdigest(query)
+        measure = Benchmark.measure do
+          CSV.open(filename, 'w') do |csv|
+            header_written = false
+            header = nil
+
+            dwh.execute_select(query) do |row|
+              unless header_written
+                header_written = true
+                header = row.keys
+                csv << header
+              end
+              csv << row.values_at(*header)
+            end
+          end
+        end
+        puts "Realizing SQL query \"#{query}\" took #{measure.real}"
+        filename
+      end
+
+      def realize_staging(params)
+        path = @options[:path]
+        url = URI.parse(path)
+        filename = Digest::SHA256.new.hexdigest(path)
+        if url.relative?
+          params['gdc_project'].download_file(path, filename)
+        else
+          params['GDC_GD_CLIENT'].download_file(path, filename)
+        end
+        filename
+      end
+
+      def realize_link
+        link = @options[:url]
+        filename = Digest::SHA256.new.hexdigest(link)
+        measure = Benchmark.measure do
+          File.open(filename, 'w') do |f|
+            open(link) { |rf| f.write(rf.read) }
+          end
+        end
+        puts "Realizing web download from \"#{link}\" took #{measure.real}"
+        filename
+      end
+
+      def realize_s3(params)
+        s3_client = params['aws_client'] && params['aws_client']['s3_client']
+        fail 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
+        bucket_name = @options[:bucket]
+        key = @options[:key]
+        fail "Key \"bucket\" is missing in S3 datasource" if bucket_name.blank?
+        fail "Key \"key\" is missing in S3 datasource" if key.blank?
+        puts "Realizing download from S3. Bucket #{bucket_name}, object with key #{key}."
+        filename = Digest::SHA256.new.hexdigest(@options.to_json)
+        bucket = s3_client.bucket(bucket_name)
+        obj = bucket.object(key)
+        File.open(filename, 'wb') do |file|
+          obj.read do |chunk|
+            file.write(chunk)
+          end
+        end
+        puts 'Done downloading file.'
+        filename
+      end
+    end
+  end
+end

data/lib/gooddata/helpers/global_helpers.rb

@@ -0,0 +1,331 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'base64'
+require 'pathname'
+require 'hashie'
+require 'openssl'
+
+require_relative 'global_helpers_params'
+
+module GoodData
+  module Helpers
+    class DeepMergeableHash < Hash
+      include Hashie::Extensions::DeepMerge
+    end
+
+    class << self
+      def error(msg)
+        STDERR.puts(msg)
+        exit 1
+      end
+
+      # FIXME: Windows incompatible
+      def find_goodfile(pwd = `pwd`.strip!, options = {})
+        root = Pathname(options[:root] || '/')
+        pwd = Pathname(pwd).expand_path
+        loop do
+          gf = pwd + '.gooddata'
+          return gf if File.exist?(gf)
+          pwd = pwd.parent
+          break if root == pwd
+        end
+        nil
+      end
+
+      # It takes what should be mapped to what and creates a mapping that is suitable for other internal methods.
+      # This means looking up the objects and returning it as array of pairs.
+      # The input can be given in several ways
+      #
+      # 1. Hash. For example it could look like
+      # {'label.states.name' => 'label.state.id'}
+      #
+      # 2 Arrays. In such case the arrays are zipped together. First item will be swapped for the first item in the second array etc.
+      # ['label.states.name'], ['label.state.id']
+      #
+      # @param what [Hash | Array] List/Hash of objects to be swapped
+      # @param for_what [Array] List of objects to be swapped
+      # @return [Array<GoodData::MdObject>] List of pairs of objects
+      def prepare_mapping(what, for_what = nil, options = {})
+        project = options[:project] || (for_what.is_a?(Hash) && for_what[:project]) || fail('Project has to be provided')
+        mapping = if what.is_a?(Hash)
+                    whats = what.keys
+                    to_whats = what.values
+                    whats.zip(to_whats)
+                  elsif what.is_a?(Array) && for_what.is_a?(Array)
+                    whats.zip(to_whats)
+                  else
+                    [[what, for_what]]
+                  end
+        mapping.pmap { |f, t| [project.objects(f), project.objects(t)] }
+      end
+
+      def get_path(an_object, path = [], default = nil)
+        return an_object if path.empty?
+        return default if an_object.nil?
+
+        path.reduce(an_object) do |a, e|
+          a && a.key?(e) ? a[e] : default
+        end
+      end
+
+      def last_uri_part(uri)
+        uri.split('/').last
+      end
+
+      def home_directory
+        running_on_windows? ? ENV['USERPROFILE'] : ENV['HOME']
+      end
+
+      def hash_dfs(thing, &block)
+        if !thing.is_a?(Hash) && !thing.is_a?(Array) # rubocop:disable Style/GuardClause
+        elsif thing.is_a?(Array)
+          thing.each do |child|
+            hash_dfs(child, &block)
+          end
+        else
+          thing.each do |key, val|
+            yield(thing, key)
+            hash_dfs(val, &block)
+          end
+        end
+      end
+
+      def titleize(str)
+        titleized = str.gsub(/[\.|_](.)/, &:upcase)
+        titleized = titleized.tr('_', ' ')
+        titleized[0] = titleized[0].upcase
+        titleized
+      end
+
+      def join(master, slave, on, on2, options = {})
+        full_outer = options[:full_outer]
+        inner = options[:inner]
+
+        lookup = create_lookup(slave, on2)
+        marked_lookup = {}
+        results = master.reduce([]) do |a, line|
+          matching_values = lookup[line.values_at(*on)] || []
+          marked_lookup[line.values_at(*on)] = 1
+          if matching_values.empty? && !inner
+            a << line.to_hash
+          else
+            matching_values.each do |matching_value|
+              a << matching_value.to_hash.merge(line.to_hash)
+            end
+          end
+          a
+        end
+
+        if full_outer
+          (lookup.keys - marked_lookup.keys).each do |key|
+            puts lookup[key]
+            results << lookup[key].first.to_hash
+          end
+        end
+        results
+      end
+
+      def running_on_windows?
+        RUBY_PLATFORM =~ /mswin32|mingw32/
+      end
+
+      def running_on_a_mac?
+        RUBY_PLATFORM =~ /-darwin\d/
+      end
+
+      def underline(x)
+        '=' * x.size
+      end
+
+      def interpolate_error_messages(errors)
+        errors.map { |e| interpolate_error_message(e) }
+      end
+
+      def interpolate_error_message(error)
+        message = error['error']['message']
+        params = error['error']['parameters']
+        sprintf(message, *params)
+      end
+
+      def transform_keys!(an_object)
+        return enum_for(:transform_keys!) unless block_given?
+        an_object.keys.each do |key|
+          an_object[yield(key)] = an_object.delete(key)
+        end
+        an_object
+      end
+
+      def symbolize_keys!(an_object)
+        transform_keys!(an_object) do |key|
+          begin
+            key.to_sym
+          rescue
+            key
+          end
+        end
+      end
+
+      def symbolize_keys(an_object)
+        transform_keys(an_object) do |key|
+          begin
+            key.to_sym
+          rescue
+            key
+          end
+        end
+      end
+
+      def transform_keys(an_object)
+        return enum_for(:transform_keys) unless block_given?
+        result = an_object.class.new
+        an_object.each_key do |key|
+          result[yield(key)] = an_object[key]
+        end
+        result
+      end
+
+      def deep_symbolize_keys(an_object)
+        deep_transform_keys(an_object) do |key|
+          begin
+            key.to_sym
+          rescue
+            key
+          end
+        end
+      end
+
+      def stringify_keys(an_object)
+        transform_keys(an_object, &:to_s)
+      end
+
+      def deep_stringify_keys(an_object)
+        deep_transform_keys(an_object, &:to_s)
+      end
+
+      def deep_transform_keys(an_object, &block)
+        _deep_transform_keys_in_object(an_object, &block)
+      end
+
+      def _deep_transform_keys_in_object(object, &block)
+        case object
+        when Hash
+          object.each_with_object({}) do |(key, value), result|
+            result[yield(key)] = _deep_transform_keys_in_object(value, &block)
+          end
+        when Array
+          object.map { |e| _deep_transform_keys_in_object(e, &block) }
+        else
+          object
+        end
+      end
+
+      def deep_dup(an_object)
+        case an_object
+        when Array
+          an_object.map { |it| GoodData::Helpers.deep_dup(it) }
+        when Hash
+          an_object.each_with_object(an_object.dup) do |(key, value), hash|
+            hash[GoodData::Helpers.deep_dup(key)] = GoodData::Helpers.deep_dup(value)
+          end
+        when Object
+          an_object.duplicable? ? an_object.dup : an_object
+        end
+      end
+
+      def undot(params)
+        # for each key-value config given
+        params.map do |k, v|
+          # dot notation to hash
+          k.split('__').reverse.reduce(v) do |memo, obj|
+            GoodData::Helper.DeepMergeableHash[{ obj => memo }]
+          end
+        end
+      end
+
+      def parse_http_exception(e)
+        JSON.parse(e.response)
+      end
+
+      # Creates a matrix with zeroes in all places. It is implemented as an Array of Arrays. First rows then columns.
+      #
+      # @param [Integer] m Number of rows
+      # @param [Integer] n Number of cols
+      # @param [Integer] val Alternatively can fill in positions with different values than zeroes. Defualt is zero.
+      # @return [Array<Array>] Returns a matrix of zeroes
+      def zeroes(m, n, val = 0)
+        m.times.map { n.times.map { val } }
+      end
+
+      # Turns a boolean or string 'true' into boolean. Useful for bricks.
+      #
+      # @param [Object] Something
+      # @return [Boolean] Returns true or false if the input is 'true' or true
+      def to_boolean(param)
+        (param == 'true' || param == true) ? true : false
+      end
+
+      # encrypts data with the given key. returns a binary data with the
+      # unhashed random iv in the first 16 bytes
+      def encrypt(data, key)
+        cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
+        cipher.encrypt
+        cipher.key = key = Digest::SHA256.digest(key)
+        random_iv = cipher.random_iv
+        cipher.iv = Digest::SHA256.digest(random_iv + key)[0..15]
+        encrypted = cipher.update(data)
+        encrypted << cipher.final
+        # add unhashed iv to front of encrypted data
+
+        Base64.encode64(random_iv + encrypted)
+      end
+
+      def decrypt(data_base_64, key)
+        return '' if key.nil? || key.empty?
+
+        data = Base64.decode64(data_base_64)
+
+        cipher = OpenSSL::Cipher::Cipher.new('aes-256-cbc')
+        cipher.decrypt
+        cipher.key = cipher_key = Digest::SHA256.digest(key)
+        random_iv = data[0..15] # extract iv from first 16 bytes
+        data = data[16..data.size - 1]
+        cipher.iv = Digest::SHA256.digest(random_iv + cipher_key)[0..15]
+        begin
+          decrypted = cipher.update(data)
+          decrypted << cipher.final
+        rescue
+          puts 'Error'
+          return nil
+        end
+
+        decrypted
+      end
+    end
+  end
+
+  class << self
+    def get_client(opts)
+      client = opts[:client]
+      fail ArgumentError, 'No :client specified' if client.nil?
+
+      client
+    end
+
+    def get_client_and_project(opts)
+      client = opts[:client]
+      fail ArgumentError, 'No :client specified' if client.nil?
+
+      p = opts[:project]
+      fail ArgumentError, 'No :project specified' if p.nil?
+
+      project = GoodData::Project[p, opts]
+      fail ArgumentError, 'Wrong :project specified' if project.nil?
+
+      [client, project]
+    end
+  end
+end

data/lib/gooddata/helpers/global_helpers_params.rb

@@ -0,0 +1,172 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  module Helpers
+    ENCODED_PARAMS_KEY = 'gd_encoded_params'
+    ENCODED_HIDDEN_PARAMS_KEY = 'gd_encoded_hidden_params'
+
+    class << self
+      # Encodes parameters for passing them to GD execution platform.
+      # Core types are kept and complex types (arrays, structures, etc) are JSON encoded into key hash "gd_encoded_params" or "gd_encoded_hidden_params", depending on the 'hidden' method param.
+      # The two different keys are used because the params and hidden params are merged by the platform and if we use the same key, the param would be overwritten.
+      #
+      # Core types are following:
+      # - Boolean (true, false)
+      # - Fixnum
+      # - Float
+      # - Nil
+      # - String
+      #
+      # @param [Hash] params Parameters to be encoded
+      # @return [Hash] Encoded parameters
+      def encode_params(params, data_key)
+        res = {}
+        nested = {}
+        core_types = [FalseClass, Fixnum, Float, NilClass, TrueClass, String]
+        params.each do |k, v|
+          if core_types.include?(v.class)
+            res[k] = v
+          else
+            nested[k] = v
+          end
+        end
+        res[data_key] = nested.to_json unless nested.empty?
+        res
+      end
+
+      # Encodes public parameters for passing them to GD execution platform.
+      # @param [Hash] params Parameters to be encoded
+      # @return [Hash] Encoded parameters
+      def encode_public_params(params)
+        encode_params(params, ENCODED_PARAMS_KEY)
+      end
+
+      # Encodes hidden parameters for passing them to GD execution platform.
+      # @param [Hash] params Parameters to be encoded
+      # @return [Hash] Encoded parameters
+      def encode_hidden_params(params)
+        encode_params(params, ENCODED_HIDDEN_PARAMS_KEY)
+      end
+
+      # Decodes params as they came from the platform
+      # The "data" key is supposed to be json and it's parsed - if this
+      def decode_params(params)
+        key = ENCODED_PARAMS_KEY.to_s
+        hidden_key = ENCODED_HIDDEN_PARAMS_KEY.to_s
+        data_params = params[key] || '{}'
+        hidden_data_params = if params.key?(hidden_key) && params[hidden_key].nil?
+                               "{\"#{hidden_key}\" : null}"
+                             elsif params.key?(hidden_key)
+                               params[hidden_key]
+                             else
+                               '{}'
+                             end
+
+        begin
+          parsed_data_params = JSON.parse(data_params)
+          parsed_hidden_data_params = JSON.parse(hidden_data_params)
+        rescue JSON::ParserError => e
+          raise e.class, "Error reading json from '#{key}' or '#{hidden_key}' in params #{params}\n #{e.message}"
+        end
+
+        # Add the nil on ENCODED_HIDDEN_PARAMS_KEY
+        # if the data was retrieved from API You will not have the actual values so encode -> decode is not losless. The nil on the key prevents the server from deleting the key
+        parsed_hidden_data_params[ENCODED_HIDDEN_PARAMS_KEY] = nil unless parsed_hidden_data_params.empty?
+        params.delete(key)
+        params.delete(hidden_key)
+        params.merge(parsed_data_params).merge(parsed_hidden_data_params)
+      end
+
+      # A helper which allows you to diff two lists of objects. The objects
+      # can be arbitrary objects as long as they respond to to_hash because
+      # the diff is eventually done on hashes. It allows you to specify
+      # several options to allow you to limit on what the sameness test is done
+      #
+      # @param [Array<Object>] old_list List of objects that serves as a base for comparison
+      # @param [Array<Object>] new_list List of objects that is compared agianst the old_list
+      # @return [Hash] A structure that contains the result of the comparison. There are
+      # four keys.
+      # :added contains the list that are in new_list but were not in the old_list
+      # :added contains the list that are in old_list but were not in the new_list
+      # :same contains objects that are in both lists and they are the same
+      # :changed contains list of objects that changed along ith original, the new one
+      # and the list of changes
+      def diff(old_list, new_list, options = {})
+        old_list = old_list.map(&:to_hash)
+        new_list = new_list.map(&:to_hash)
+
+        fields = options[:fields]
+        lookup_key = options[:key]
+
+        old_lookup = Hash[old_list.map { |v| [v[lookup_key], v] }]
+
+        res = {
+          :added => [],
+          :removed => [],
+          :changed => [],
+          :same => []
+        }
+
+        new_list.each do |new_obj|
+          old_obj = old_lookup[new_obj[lookup_key]]
+          if old_obj.nil?
+            res[:added] << new_obj
+            next
+          end
+
+          if fields
+            sliced_old_obj = old_obj.slice(*fields)
+            sliced_new_obj = new_obj.slice(*fields)
+          else
+            sliced_old_obj = old_obj
+            sliced_new_obj = new_obj
+          end
+          if sliced_old_obj != sliced_new_obj
+            difference = sliced_new_obj.to_a - sliced_old_obj.to_a
+            differences = Hash[*difference.mapcat { |x| x }]
+            res[:changed] << {
+              old_obj: old_obj,
+              new_obj: new_obj,
+              diff: differences
+            }
+          else
+            res[:same] << old_obj
+          end
+        end
+
+        new_lookup = Hash[new_list.map { |v| [v[lookup_key], v] }]
+        old_list.each do |old_obj|
+          new_obj = new_lookup[old_obj[lookup_key]]
+          if new_obj.nil?
+            res[:removed] << old_obj
+            next
+          end
+        end
+
+        res
+      end
+
+      def create_lookup(collection, on)
+        lookup = {}
+        if on.is_a?(Array)
+          collection.each do |e|
+            key = e.values_at(*on)
+            lookup[key] = [] unless lookup.key?(key)
+            lookup[key] << e
+          end
+        else
+          collection.each do |e|
+            key = e[on]
+            lookup[key] = [] unless lookup.key?(key)
+            lookup[key] << e
+          end
+        end
+        lookup
+      end
+    end
+  end
+end