gooddata 0.6.18 → 0.6.19

data/lib/gooddata/models/metadata.rb

@@ -38,6 +38,11 @@ module GoodData
       @json = data.to_hash
     end
 
+    def add_tag(a_tag)
+      self.tags = tag_set.add(a_tag).to_a.join(' ')
+      self
+    end
+
     def delete
       if saved? # rubocop:disable Style/GuardClause
         client.delete(uri)
@@ -70,6 +75,11 @@ module GoodData
       @project ||= Project[uri.gsub(%r{\/obj\/\d+$}, ''), :client => client]
     end
 
+    def remove_tag(a_tag)
+      self.tags = tag_set.delete(a_tag).to_a.join(' ')
+      self
+    end
+
     def saved?
       res = uri.nil?
       !res
@@ -132,6 +142,10 @@ module GoodData
       x.save
     end
 
+    def tag_set
+      tags.scan(/\w+/).to_set
+    end
+
     def ==(other)
       other.respond_to?(:uri) && other.uri == uri && other.respond_to?(:to_hash) && other.to_hash == to_hash
     end

data/lib/gooddata/models/metadata/dashboard.rb

@@ -88,9 +88,9 @@ module GoodData
       fail "Wrong format provied \"#{format}\". Only supports formats #{supported_formats.join(', ')}" unless supported_formats.include?(format)
       tab = options[:tab] || ''
 
-      req_uri = "/gdc/projects/#{GoodData.project.pid}/clientexport"
+      req_uri = "/gdc/projects/#{project.pid}/clientexport"
       x = client.post(req_uri, 'clientExport' => { 'url' => "https://secure.gooddata.com/dashboard.html#project=#{GoodData.project.uri}&dashboard=#{uri}&tab=#{tab}&export=1", 'name' => title })
-      GoodData.poll_on_code(x['asyncTask']['link']['poll'], process: false)
+      client.poll_on_code(x['asyncTask']['link']['poll'], options.merge(process: false))
     end
 
     def tabs

data/lib/gooddata/models/metadata/label.rb

@@ -33,7 +33,7 @@ module GoodData
       result = client.get(uri + "/?id=#{element_id}")
       items = result['attributeElements']['elements']
       if items.empty?
-        fail "Element id #{element_id} was not found"
+        fail(AttributeElementNotFound, element_id)
       else
         items.first['title']
       end

data/lib/gooddata/models/metadata/report.rb

@@ -26,7 +26,7 @@ module GoodData
         p = options[:project]
         fail ArgumentError, 'No :project specified' if p.nil?
 
-        project = GoodData::Project[p, options]
+        project = client.projects(p)
         fail ArgumentError, 'Wrong :project specified' if project.nil?
 
         title = options[:title]
@@ -71,6 +71,7 @@ module GoodData
     def definitions
       content['definitions'].pmap { |uri| project.report_definitions(uri) }
     end
+    alias_method :report_definitions, :definitions
 
     # Gets list of uris of report definitions (versions) of this report.
     #
@@ -92,12 +93,12 @@ module GoodData
     # Computes the report and returns the result. If it is not computable returns nil.
     #
     # @return [GoodData::DataResult] Returns the result
-    def execute
+    def execute(options = {})
       fail 'You have to save the report before executing. If you do not want to do that please use GoodData::ReportDefinition' unless saved?
       result = client.post '/gdc/xtab2/executor3', 'report_req' => { 'report' => uri }
       data_result_uri = result['execResult']['dataResult']
 
-      result = client.poll_on_response(data_result_uri) do |body|
+      result = client.poll_on_response(data_result_uri, options) do |body|
         body && body['taskState'] && body['taskState']['status'] == 'WAIT'
       end
 
@@ -119,10 +120,10 @@ module GoodData
     # either 'csv', 'xls', 'xlsx' or 'pdf'.
     #
     # @return [String] Returns data
-    def export(format)
+    def export(format, options = {})
       result = client.post('/gdc/xtab2/executor3', 'report_req' => { 'report' => uri })
       result1 = client.post('/gdc/exporter/executor', :result_req => { :format => format, :result => result })
-      GoodData.poll_on_code(result1['uri'], process: false)
+      client.poll_on_code(result1['uri'], options.merge(process: false))
     end
 
     # Returns the newest (current version) report definition uri

data/lib/gooddata/models/metadata/report_definition.rb

@@ -131,57 +131,12 @@ module GoodData
 
         begin
           unsaved_metrics.each(&:save)
-          rd = GoodData::ReportDefinition.create(options)
-          data_result(execute_inline(rd, options), options)
+          GoodData::ReportDefinition.create(options).execute
         ensure
           unsaved_metrics.each { |m| m.delete if m && m.saved? }
         end
       end
 
-      def execute_inline(rd, opts = { :client => GoodData.connection, :project => GoodData.project })
-        client = opts[:client]
-        project = opts[:project]
-
-        rd = rd.respond_to?(:json) ? rd.json : rd
-        data = {
-          report_req: {
-            definitionContent: {
-              content: rd,
-              projectMetadata: project.links['metadata']
-            }
-          }
-        }
-        uri = "/gdc/app/projects/#{project.pid}/execute"
-        client.post(uri, data)
-      end
-
-      # TODO: refactor the method. It should be instance method
-      # Method used for getting a data_result from a wire representation of
-      # @param result [Hash, Object] Wire data from JSON
-      # @return [GoodData::ReportDataResult]
-      def data_result(result, options = { :client => GoodData.connection })
-        client = options[:client]
-        fail ArgumentError, 'No :client specified' if client.nil?
-
-        data_result_uri = result['execResult']['dataResult']
-        result = client.poll_on_response(data_result_uri) do |body|
-          body && body['taskState'] && body['taskState']['status'] == 'WAIT'
-        end
-
-        if result.empty?
-          client.create(EmptyResult, result)
-        else
-          client.create(ReportDataResult, result)
-        end
-      end
-
-      # Return true if the report definition is a chart
-      #
-      # @return [Boolean] Return true if report definition is a chart
-      def chart?
-        !table?
-      end
-
       def create(options = { :client => GoodData.connection, :project => GoodData.project })
         client = options[:client]
         fail ArgumentError, 'No :client specified' if client.nil?
@@ -253,6 +208,13 @@ module GoodData
       self
     end
 
+    # Return true if the report definition is a chart
+    #
+    # @return [Boolean] Return true if report definition is a chart
+    def chart?
+      !table?
+    end
+
     def labels
       attribute_parts.map { |part| project.labels(part['attribute']['uri']) }
     end
@@ -265,26 +227,25 @@ module GoodData
       metric_parts.map { |i| project.metrics(i['uri']) }
     end
 
-    def execute(opts = { :client => GoodData.connection, :project => GoodData.project })
-      client = opts[:client]
-      fail ArgumentError, 'No :client specified' if client.nil?
-
-      p = opts[:project]
-      fail ArgumentError, 'No :project specified' if p.nil?
-
-      project = client.projects(p)
-      fail ArgumentError, 'Wrong :project specified' if project.nil?
-
-      opts = { client: client, project: project }
+    def execute(opts = {})
       result = if saved?
                  pars = {
                    'report_req' => { 'reportDefinition' => uri }
                  }
                  client.post '/gdc/xtab2/executor3', pars
                else
-                 ReportDefinition.execute_inline(self, opts)
+                 data = {
+                   report_req: {
+                     definitionContent: {
+                       content: to_hash,
+                       projectMetadata: project.links['metadata']
+                     }
+                   }
+                 }
+                 uri = "/gdc/app/projects/#{project.pid}/execute"
+                 client.post(uri, data)
                end
-      ReportDefinition.data_result(result, opts)
+      data_result(result, opts)
     end
 
     def filters
@@ -394,5 +355,29 @@ module GoodData
     def table?
       content['format'] == 'grid'
     end
+
+    private
+
+    def data_result(result, options = {})
+      data_result_uri = result['execResult']['dataResult']
+      begin
+        result = client.poll_on_response(data_result_uri, options) do |body|
+          body && body['taskState'] && body['taskState']['status'] == 'WAIT'
+        end
+      rescue RestClient::BadRequest => e
+        resp = JSON.parse(e.response)
+        if GoodData::Helpers.get_path(resp, %w(error component)) == 'MD::DataResult'
+          raise GoodData::UncomputableReport
+        else
+          raise e
+        end
+      end
+
+      if result.empty?
+        client.create(EmptyResult, result)
+      else
+        client.create(ReportDataResult, result)
+      end
+    end
   end
 end

data/lib/gooddata/models/model.rb

@@ -7,6 +7,7 @@ require_relative 'metadata/metadata'
 require_relative 'links'
 require_relative 'module_constants'
 require_relative 'user_filters/user_filters'
+require_relative 'blueprint/blueprint'
 
 require 'fileutils'
 require 'multi_json'
@@ -19,53 +20,113 @@ require 'zip'
 #
 module GoodData
   module Model
-    GD_TYPES = %w(GDC.link GDC.text GDC.geo GDC.time)
-    GD_DATA_TYPES = %w(INT VARCHAR DECIMAL)
-
-    DEFAULT_FACT_DATATYPE = 'INT'
+    # See https://confluence.intgdc.com/display/plat/Catalog+of+Attribute+Types
+    GD_TYPES = [
+      # Common Types
+      'GDC.link',
+      'GDC.text',
+      'GDC.time',
+
+      # Common Date Attribute Types
+      'GDC.time.year',
+      'GDC.time.quarter',
+      'GDC.time.month',
+      'GDC.time.week',
+      'GDC.time.date',
+
+      # Specific Date Attribute Types
+      'GDC.time.day_in_euweek',
+      'GDC.time.day_in_week',
+      'GDC.time.day_in_month',
+      'GDC.time.day_in_quarter',
+      'GDC.time.day_in_year',
+      'GDC.time.euweek_in_quarter',
+      'GDC.time.week_in_quarter',
+      'GDC.time.euweek_in_year',
+      'GDC.time.week_in_year',
+      'GDC.time.month_in_quarter',
+      'GDC.time.month_in_year',
+      'GDC.time.quarter_in_year',
+
+      # Legacy Date Attribute Types - Possibly Obsolete
+      'GDC.time.dayOfWeek',
+      'GDC.time.dayOfMonth',
+      'GDC.time.dayOfQuarter',
+      'GDC.time.dayOfYear',
+      'GDC.time.weekOfYear',
+      'GDC.time.monthOfYear',
+      'GDC.time.quarterOfYear',
+
+      # Types for Geo
+      'GDC.geo.pin',                 # Geo pushpin
+      'GDC.geo.ausstates.name',      # Australia States (Name)
+      'GDC.geo.ausstates.code',      # Australia States (ISO code)
+      'GDC.geo.usstates.name',       # US States (Name)
+      'GDC.geo.usstates.geo_id',     # US States (US Census ID)
+      'GDC.geo.usstates.code',       # US States (2-letter code)
+      'GDC.geo.uscounties.geo_id',   # US Counties (US Census ID)
+      'GDC.geo.worldcountries.name', # World countries (Name)
+      'GDC.geo.worldcountries.iso2', # World countries (ISO a2)
+      'GDC.geo.worldcountries.iso3', # World countries (ISO a3)
+      'GDC.geo.czdistricts.name',    #	Czech Districts (Name)
+      'GDC.geo.czdistricts.name_no_diacritics', # Czech Districts
+      'GDC.geo.czdistricts.nuts4',   # Czech Districts (NUTS 4)
+      'GDC.geo.czdistricts.knok',    # Czech Districts (KNOK)
+
+      # Day Display Forms
+      'GDC.time.day',              # yyyy-MM-dd
+      'GDC.time.day_us',           # MM/dd/yyyy
+      'GDC.time.day_eu',           # dd/MM/yyyy
+      'GDC.time.day_iso',          # dd-MM-yyyy
+      'GDC.time.day_us_long',      # EEE, MMM d, yyyy
+      'GDC.time.day_us_noleading', # M/d/yy
+    ]
+
+    GD_DATA_TYPES = ['BIGINT', 'DOUBLE', 'INTEGER', 'INT', /^VARCHAR\(\d{1,3}\)$/i, /^DECIMAL\(\d{1,3},\s*\d{1,3}\)$/i]
+
+    DEFAULT_FACT_DATATYPE = 'DECIMAL(12,2)'
+    DEFAULT_ATTRIBUTE_DATATYPE = 'VARCHAR(128)'
+
+    DEFAULT_TYPE = 'GDC.text'
+
+    DEFAULT_DATE_FORMAT = 'MM/dd/yyyy'
 
     class << self
       def title(item)
-        item[:title] || item[:name].titleize
+        item[:title] || item[:id].titleize
+      end
+
+      def column_name(item)
+        item[:column_name] || item[:id]
       end
 
       def description(item)
         item[:description]
       end
 
-      def identifier_for(dataset, column = nil, column2 = nil) # rubocop:disable UnusedMethodArgument
-        return "dataset.#{dataset[:name]}" if column.nil?
-        column = DatasetBlueprint.find_column_by_name(dataset, column) if column.is_a?(String)
-        case column[:type].to_sym
-        when :anchor_no_label
-          "attr.#{dataset[:name]}.factsof"
-        when :attribute
-          "attr.#{dataset[:name]}.#{column[:name]}"
-        when :anchor
-          "attr.#{dataset[:name]}.#{column[:name]}"
-        when :date_fact
-          "dt.#{dataset[:name]}.#{column[:name]}"
-        when :fact
-          "fact.#{dataset[:name]}.#{column[:name]}"
-        when :primary_label
-          "label.#{dataset[:name]}.#{column[:name]}"
-        when :label
-          "label.#{dataset[:name]}.#{column[:reference]}.#{column[:name]}"
-        when :date_ref
-          "#{dataset[:name]}.date.mdyy"
-        when :dataset
-          "dataset.#{dataset[:name]}"
-        when :date
-          'DATE'
-        when :reference
-          'REF'
-        else
-          fail "Unknown type #{column[:type].to_sym}"
+      def check_gd_type(value)
+        GD_TYPES.any? { |v| v == value }
+      end
+
+      def check_gd_data_type(value)
+        GD_DATA_TYPES.any? do |v|
+          case v
+          when Regexp
+            v =~ value
+          when String
+            v == (value && value.upcase)
+          else
+            fail 'Unkown predicate'
+          end
         end
       end
 
-      def check_gd_datatype(value)
-        GD_TYPES.any? { |v| v == value }
+      def normalize_gd_data_type(type)
+        if type && type.upcase == 'INTEGER'
+          'INT'
+        else
+          type
+        end
       end
 
       # Load given file into a data set described by the given schema
@@ -96,6 +157,7 @@ module GoodData
 
         path = path.path if path.respond_to? :path
         inline_data = path.is_a?(String) ? false : true
+        csv_header = nil
 
         # create a temporary zip file
         dir = Dir.mktmpdir
@@ -104,12 +166,14 @@ module GoodData
             # TODO: make sure schema columns match CSV column names
             zip.get_output_stream('upload_info.json') { |f| f.puts JSON.pretty_generate(manifest) }
             if inline_data
+              csv_header = path.first
               zip.get_output_stream('data.csv') do |f|
                 path.each do |row|
                   f.puts row.to_csv
                 end
               end
             else
+              csv_header = File.open(path, &:gets).split(',')
               zip.add('data.csv', path)
             end
           end
@@ -122,18 +186,36 @@ module GoodData
 
         # kick the load
         pull = { 'pullIntegration' => File.basename(dir) }
-        link = project.md.links('etl')['pull']
-        task = client.post link, pull
+        link = project.md.links('etl')['pull2']
+        task = client.post(link, pull, :info_message => "Starting the data load from user storage to dataset '#{dataset}'.")
 
-        res = client.poll_on_response(task['pullTask']['uri']) do |body|
-          body['taskStatus'] == 'RUNNING' || body['taskStatus'] == 'PREPARED'
+        res = client.poll_on_response(task['pull2Task']['links']['poll'], :info_message => 'Getting status of the dataload task.') do |body|
+          body['wTaskStatus']['status'] == 'RUNNING' || body['wTaskStatus']['status'] == 'PREPARED'
         end
 
-        if res['taskStatus'] == 'ERROR' # rubocop:disable Style/GuardClause
+        if res['wTaskStatus']['status'] == 'ERROR' # rubocop:disable Style/GuardClause
           s = StringIO.new
           client.download_from_user_webdav(File.basename(dir) + '/upload_status.json', s, :client => client, :project => project)
           js = MultiJson.load(s.string)
-          fail "Load Failed with error #{JSON.pretty_generate(js)}"
+          manifest_cols =  manifest['dataSetSLIManifest']['parts'].map { |c| c['columnName'] }
+
+          # extract some human readable error message from the webdav file
+          manifest_extra = manifest_cols - csv_header
+          csv_extra = csv_header - manifest_cols
+
+          error_message = begin
+                            js['error']['message'] % js['error']['parameters']
+                          rescue NoMethodError, ArgumentError
+                            ''
+                          end
+          m = "Load failed with error '#{error_message}'.\n"
+          m += "Columns that should be there (manifest) but aren't in uploaded csv: #{manifest_extra}\n" unless manifest_extra.empty?
+          m += "Columns that are in csv but shouldn't be there (manifest): #{csv_extra}\n" unless csv_extra.empty?
+          m += "Columns in the uploaded csv: #{csv_header}\n"
+          m += "Columns in the manifest: #{manifest_cols}\n"
+          m += "Original message:\n#{JSON.pretty_generate(js)}\n"
+          m += "Manifest used for uploading:\n#{JSON.pretty_generate(manifest)}"
+          fail m
         end
       end
 
@@ -143,8 +225,14 @@ module GoodData
         d = a_schema_blueprint.deep_dup
         d[:columns] = d[:columns] + b_schema_blueprint[:columns]
         d[:columns].uniq!
-        columns_that_failed_to_merge = d[:columns].group_by { |x| x[:name] }.map { |k, v| [k, v.count] }.select { |x| x[1] > 1 }
-        fail "Columns #{columns_that_failed_to_merge} failed to merge. When merging columns with the same name they have to be identical." unless columns_that_failed_to_merge.empty?
+        columns_that_failed_to_merge = d[:columns].group_by { |x| [:reference, :date].include?(x[:type]) ? x[:dataset] : x[:id] }.map { |k, v| [k, v.count, v] }.select { |x| x[1] > 1 }
+        unless columns_that_failed_to_merge.empty?
+          columns_that_failed_to_merge.each do |error|
+            GoodData.logger.error "Columns #{error[0]} failed to merge. There are #{error[1]} conflicting columns. When merging columns with the same name they have to be identical."
+            GoodData.logger.error error[2]
+          end
+          fail "Columns #{columns_that_failed_to_merge.first} failed to merge. There are #{columns_that_failed_to_merge[1]} conflicting columns. #{columns_that_failed_to_merge[2]} When merging columns with the same name they have to be identical." unless columns_that_failed_to_merge.empty?
+        end
         d
       end
     end