gooddata 1.0.0-java

data/lib/gooddata/extensions/numeric.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+class Numeric
+  # Numbers are not duplicable:
+  #
+  #  3.duplicable? # => false
+  #  3.dup         # => TypeError: can't dup Fixnum
+  def duplicable?
+    false
+  end
+end

data/lib/gooddata/extensions/object.rb

@@ -0,0 +1,31 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+class Object
+  class << self
+    def set_const(name, val)
+      send(:remove_const, name) if const_defined?(name)
+      send(:const_set, name, val)
+    end
+  end
+
+  def blank?
+    respond_to?(:empty?) ? empty? : !self
+  end
+
+  def duplicable?
+    true
+  end
+
+  def set_const(name, val)
+    send(:remove_const, name) if const_defined?(name)
+    send(:const_set, name, val)
+  end
+
+  def to_b
+    false
+  end
+end

data/lib/gooddata/extensions/string.rb

@@ -0,0 +1,7 @@
+class String
+  def to_b
+    return true if self == true || self =~ (/(true|t|yes|y|1)$/i)
+    return false if self == false || blank? || self =~ (/(false|f|no|n|0)$/i)
+    raise ArgumentError, "invalid value for Boolean: \"#{self}\""
+  end
+end

data/lib/gooddata/extensions/symbol.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+class Symbol
+  # Symbols are not duplicable:
+  #
+  #   :my_symbol.duplicable? # => false
+  #   :my_symbol.dup         # => TypeError: can't dup Symbol
+  def duplicable?
+    false
+  end
+end

data/lib/gooddata/extensions/true.rb

@@ -0,0 +1,23 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+class TrueClass
+  # +true+ is not duplicable:
+  #
+  #   true.duplicable? # => false
+  #   true.dup         # => TypeError: can't dup TrueClass
+  def duplicable?
+    false
+  end
+
+  def to_b
+    true
+  end
+
+  def to_i
+    1
+  end
+end

data/lib/gooddata/extract.rb

@@ -0,0 +1,21 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'csv'
+
+module GoodData
+  module Extract
+    class CsvFile
+      def initialize(file)
+        @file = file
+      end
+
+      def read(&block)
+        CSV.open @file, 'r', &block
+      end
+    end
+  end
+end

data/lib/gooddata/goodzilla/goodzilla.rb

@@ -0,0 +1,160 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  module SmallGoodZilla
+    class << self
+      # Scans the provided MAQL and returns Array pairs of [attribute, element] pairs for each element that is found in the definition
+      # @param maql Input MAQL string
+      # @return [Array<Array>] Pairs [attribute, attribute_element]
+      def extract_element_uri_pairs(maql)
+        arr = maql.scan(%r{(\/gdc\/(?:projects|md)\/[a-zA-Z\d]+\/obj\/\d+)\/elements\?id=(\d+)}).flatten
+        evens = arr.select.each_with_index { |_, i| i.even? }
+        odds = arr.select.each_with_index { |_, i| i.odd? }.map(&:to_i)
+        evens.zip(odds)
+      end
+
+      # Scans the provided MAQL and returns Array of all the URIs included in the MAQL. This basically return anything that is enclosed in aquare brackets []
+      # @param maql Input MAQL string
+      # @return [Array<String>] Pairs of URIs
+      def get_uris(a_maql_string)
+        a_maql_string.scan(/\[([^\"\]]+)\]/).flatten.uniq
+      end
+
+      # Get IDs from extendedMAQL string
+      # @param a_maql_string Input MAQL string
+      # @return [Array<String>] List of IDS
+      def get_ids(a_maql_string)
+        a_maql_string.scan(/!\[([^\"\]]+)\]/).flatten.uniq
+      end
+
+      # Get Facts from extendedMAQL string
+      # @param a_maql_string Input MAQL string
+      # @return [Array<String>] List of Facts
+      def get_facts(a_maql_string)
+        a_maql_string.scan(/#\"([^\"]+)\"/).flatten
+      end
+
+      # Get Attributes from extendedMAQL string
+      # @param a_maql_string Input MAQL string
+      # @return [Array<String>] List of Attributes
+      def get_attributes(a_maql_string)
+        a_maql_string.scan(/@\"([^\"]+)\"/).flatten
+      end
+
+      # Get Metrics from extendedMAQL string
+      # @param a_maql_string Input MAQL string
+      # @return [Array<String>] List of Metrics
+      def get_metrics(a_maql_string)
+        a_maql_string.scan(/\?"([^\"]+)\"/).flatten
+      end
+
+      alias_method :get_measures, :get_metrics
+
+      # Method takes a specification of the attribute filter (category filter) and returns it representation that is
+      # suitable for posting on the API. The spec is expected to be an array. First object can be an attribute (id, obj_id or
+      # directly an object). Alternativel it can be an attribute (again any representation should work). In case of attribute
+      # primary label is taken. The rest of the array are expected to be String represenation of values of particular label.
+      #
+      # For example it could look like
+      # ['label.states.name', 'California', 'New Jersey', 'Kansas']
+      #
+      # @param spec [Array<Object>] Input MAQL string
+      # @return [Array<Hash>] List of Metrics
+      def create_category_filter(spec, project)
+        item = project.objects(spec.first)
+        label = item.is_a?(GoodData::Attribute) ? item.primary_label : item
+        col = spec[1..-1].flat_map do |v|
+          case v
+          when Range
+            v.to_a
+          when Symbol
+            [v]
+          else
+            [v.to_s]
+          end
+        end
+        if col.first == :not
+          values = col[1..-1].map { |v| label.find_value_uri(v) }
+          elements = values.map { |v| "[#{v}]" }.join(', ')
+          { expression: "[#{label.attribute.uri}] NOT IN (#{elements})" }
+        else
+          values = col.map { |v| label.find_value_uri(v) }
+          elements = values.map { |v| "[#{v}]" }.join(', ')
+          { expression: "[#{label.attribute.uri}] IN (#{elements})" }
+        end
+      end
+
+      # Pretty prints the MAQL expression. This basically means it finds out names of objects and elements and print their values instead of URIs
+      # @param expression [String] Expression to be beautified
+      # @return [String] Pretty printed MAQL expression
+      def pretty_print(expression, opts = { client: GoodData.connection, project: GoodData.project })
+        temp = expression.dup
+        pairs = get_uris(expression).pmap do |uri|
+          if uri =~ /elements/
+            begin
+              ['element', uri, Attribute.find_element_value(uri, opts)]
+            rescue AttributeElementNotFound
+              ['element', uri, '(empty value)']
+            end
+          else
+            ['object', uri, GoodData::MdObject[uri, opts].title]
+          end
+        end
+        pairs.sort_by! { |p| p[0] }
+        pairs.each do |el|
+          uri = el[1]
+          obj = el[2]
+          temp.gsub!(uri, obj)
+        end
+        temp
+      end
+
+      def interpolate(values, dictionaries)
+        {
+          :facts => interpolate_values(values[:facts], dictionaries[:facts]),
+          :attributes => interpolate_values(values[:attributes], dictionaries[:attributes]),
+          :metrics => interpolate_values(values[:metrics], dictionaries[:metrics])
+        }
+      end
+
+      def interpolate_ids(options, *ids)
+        ids = ids.flatten
+        if ids.empty?
+          []
+        else
+          res = GoodData::MdObject.identifier_to_uri(options, *ids)
+          fail 'Not all of the identifiers were resolved' if Array(res).size != ids.size
+          res
+        end
+      end
+
+      def interpolate_values(keys, values)
+        x = values.values_at(*keys)
+        keys.zip(x)
+      end
+
+      def interpolate_metric(metric, dictionary, options = { :client => GoodData.connection, :project => GoodData.project })
+        interpolated = interpolate({
+                                     :facts => GoodData::SmallGoodZilla.get_facts(metric),
+                                     :attributes => GoodData::SmallGoodZilla.get_attributes(metric),
+                                     :metrics => GoodData::SmallGoodZilla.get_metrics(metric)
+                                   }, dictionary)
+
+        ids = GoodData::SmallGoodZilla.get_ids(metric)
+        interpolated_ids = ids.zip(Array(interpolate_ids(options, ids)))
+
+        metric = interpolated[:facts].reduce(metric) { |acc, elem| acc.sub("#\"#{elem[0]}\"", "[#{elem[1]}]") }
+        metric = interpolated[:attributes].reduce(metric) { |acc, elem| acc.sub("@\"#{elem[0]}\"", "[#{elem[1]}]") }
+        metric = interpolated[:metrics].reduce(metric) { |acc, elem| acc.sub("?\"#{elem[0]}\"", "[#{elem[1]}]") }
+        metric = interpolated_ids.reduce(metric) { |acc, elem| acc.gsub("![#{elem[0]}]", "[#{elem[1]}]") }
+        metric
+      end
+
+      alias_method :interpolate_measure, :interpolate_metric
+    end
+  end
+end

data/lib/gooddata/helpers/auth_helpers.rb

@@ -0,0 +1,75 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'fileutils'
+require 'json'
+
+require_relative 'global_helpers'
+
+module GoodData
+  module Helpers
+    module AuthHelper
+      class << self
+        # Get path of .gooddata config
+        def credentials_file
+          "#{Helpers.home_directory}/.gooddata"
+        end
+
+        # Read credentials
+        def read_credentials(credentials_file_path = credentials_file)
+          if File.exist?(credentials_file_path)
+            config = File.read(credentials_file_path)
+            MultiJson.load(config, :symbolize_keys => true)
+          else
+            {}
+          end
+        end
+
+        # Try read environemnt
+        #
+        # Tries to read it from ~/.gooddata file or from environment variable GD_SERVER
+        # @param [String] credentials_file_path (credentials_file) Path to .gooddata file
+        # @return [String] server token from environment variable, .gooddata or nil
+        def read_environment(credentials_file_path = credentials_file)
+          goodfile = read_credentials(credentials_file_path)
+          [ENV['GD_ENVIRONMENT'], goodfile[:environment], GoodData::Project::DEFAULT_ENVIRONMENT].find { |x| !x.nil? && !x.empty? }
+        end
+
+        # Try read server
+        #
+        # Tries to read it from ~/.gooddata file or from environment variable GD_SERVER
+        # @param [String] credentials_file_path (credentials_file) Path to .gooddata file
+        # @return [String] server token from environment variable, .gooddata or DEFAULT_URL
+        def read_server(credentials_file_path = credentials_file)
+          goodfile = read_credentials(credentials_file_path)
+          [ENV['GD_SERVER'], goodfile[:server], GoodData::Rest::Connection::DEFAULT_URL].find { |x| !x.nil? && !x.empty? }
+        end
+
+        # Try read token
+        #
+        # Tries to read it from ~/.gooddata file or from environment variable GD_PROJECT_TOKEN
+        # @param [String] credentials_file_path (credentials_file) Path to .gooddata file
+        # @return [String] auth token from environment variable, .gooddata or nil
+        def read_token(credentials_file_path = credentials_file)
+          goodfile = read_credentials(credentials_file_path)
+          [ENV['GD_PROJECT_TOKEN'], goodfile[:auth_token], goodfile[:token]].find { |x| !x.nil? && !x.empty? }
+        end
+
+        # Writes credentials
+        def write_credentials(credentials, credentials_file_path = credentials_file)
+          File.open(credentials_file_path, 'w', 0o600) do |f|
+            f.puts JSON.pretty_generate(credentials)
+          end
+          credentials
+        end
+
+        def remove_credentials_file(credentials_file_path = credentials_file)
+          FileUtils.rm_f(credentials_file_path)
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/helpers/csv_helper.rb

@@ -0,0 +1,61 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'csv'
+
+module GoodData
+  module Helpers
+    class Csv
+      class << self
+        # Read data from CSV
+        #
+        # @param [Hash] opts
+        # @option opts [String] :path File to read data from
+        # @option opts [Boolean] :header File to read data from
+        # @return Array of rows with loaded data
+        def read(opts)
+          path = opts[:path]
+          res = []
+
+          line = 0
+
+          CSV.foreach(path) do |row|
+            line += 1
+            next if opts[:header] && line == 1
+
+            if block_given?
+              data = yield row
+            else
+              data = row
+            end
+
+            res << data if data
+          end
+
+          res
+        end
+
+        # Write data to CSV
+        # @option opts [String] :path File to write data to
+        # @option opts [Array] :data Mandatory array of data to write
+        # @option opts [String] :header Optional Header row
+        def write(opts, &_block)
+          path = opts[:path]
+          header = opts[:header]
+          data = opts[:data]
+
+          CSV.open(path, 'w') do |csv|
+            csv << header unless header.nil?
+            data.each do |entry|
+              res = yield entry
+              csv << res if res
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/helpers/data_helper.rb

@@ -0,0 +1,129 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'csv'
+require 'digest'
+require 'open-uri'
+
+module GoodData
+  module Helpers
+    class DataSource
+      attr_reader :realized
+
+      class << self
+        def interpolate_sql_params(query, params)
+          keys = query.scan(/\$\{([^\{]+)\}/).flatten
+          keys.reduce(query) do |a, e|
+            key = e
+            raise "Param #{key} is not present in schedule params yet it is expected to be interpolated in the query" unless params.key?(key)
+            a.gsub("${#{key}}", params[key])
+          end
+        end
+      end
+
+      def initialize(opts = {})
+        opts = opts.is_a?(String) ? { type: :staging, path: opts } : opts
+        opts = GoodData::Helpers.symbolize_keys(opts)
+        @source = opts[:type]
+        @options = opts
+        @realized = false
+      end
+
+      def realize(params = {})
+        @realized = true
+        source = @source && @source.to_s
+        case source
+        when 'ads'
+          realize_query(params)
+        when 'staging'
+          realize_staging(params)
+        when 'web'
+          realize_link
+        when 's3'
+          realize_s3(params)
+        else
+          raise "DataSource does not support type \"#{source}\""
+        end
+      end
+
+      def realized?
+        @realized == true
+      end
+
+      private
+
+      def realize_query(params)
+        query = DataSource.interpolate_sql_params(@options[:query], params)
+        dwh = params['ads_client'] || params[:ads_client] || raise("Data Source needs a client to ads to be able to query the storage but 'ads_client' is empty.")
+        filename = Digest::SHA256.new.hexdigest(query)
+        measure = Benchmark.measure do
+          CSV.open(filename, 'w') do |csv|
+            header_written = false
+            header = nil
+            dwh.execute_select(query) do |row|
+              unless header_written
+                header_written = true
+                header = row.keys
+                csv << header
+              end
+              csv << row.values_at(*header)
+            end
+          end
+        end
+        GoodData.logger.info "Realizing SQL query \"#{query}\" took #{measure.real}"
+        filename
+      end
+
+      def realize_staging(params)
+        path = @options[:path]
+        url = URI.parse(path)
+        filename = Digest::SHA256.new.hexdigest(path)
+        if url.relative?
+          params['gdc_project'].download_file(path, filename)
+        else
+          params['GDC_GD_CLIENT'].download_file(path, filename)
+        end
+        filename
+      end
+
+      def realize_link
+        link = @options[:url]
+        filename = Digest::SHA256.new.hexdigest(link)
+        measure = Benchmark.measure do
+          File.open(filename, 'w') do |f|
+            open(link) { |rf| f.write(rf.read) }
+          end
+        end
+        GoodData.logger.info("Realizing web download from \"#{link}\" took #{measure.real}")
+        filename
+      end
+
+      def realize_s3(params)
+        params = GoodData::Helpers.stringify_keys(params)
+        s3_client = params['aws_client'] && params['aws_client']['s3_client']
+        raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:bucket)
+        bucket_name = @options[:bucket]
+        key = @options[:key]
+        raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
+        raise 'Key "key" is missing in S3 datasource' if key.blank?
+        GoodData.logger.info("Realizing download from S3. Bucket #{bucket_name}, object with key #{key}.")
+        filename = Digest::SHA256.new.hexdigest(@options.to_json)
+        bucket = s3_client.bucket(bucket_name)
+        obj = bucket.object(key)
+        obj.get(response_target: filename, bucket: bucket_name, key: key)
+        s3_size = obj.size
+        actual_size = File.size(filename)
+        GoodData.logger.info("File size in S3: #{s3_size}")
+        GoodData.logger.info("Downloaded file size: #{actual_size}")
+        unless s3_size == actual_size
+          fail "Error downloading file #{key}. Expected size #{s3_size}, got #{actual_size}."
+        end
+        GoodData.logger.info('Done downloading file.')
+        filename
+      end
+    end
+  end
+end