gooddata 0.2.0 → 0.4.0

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.4.0

data/lib/gooddata.rb

@@ -1,3 +1,4 @@
 module GoodData; end
 
 require 'gooddata/client'
+require 'gooddata/model'

data/lib/gooddata/client.rb

@@ -72,8 +72,8 @@ module GoodData
     # * +user+ - A GoodData username
     # * +password+ - A GoodData password
     #
-    def connect(user, password, url = nil)
-      threaded[:connection] = Connection.new user, password, url
+    def connect(user, password, url = nil, options={})
+      threaded[:connection] = Connection.new user, password, url, options
     end
 
     # Returns the active GoodData connection earlier initialized via

data/lib/gooddata/connection.rb

@@ -1,4 +1,4 @@
-require 'json/pure'
+require 'json'
 require 'net/ftptls'
 
 # silence the parenthesis warning in rest-client 1.6.1
@@ -42,11 +42,12 @@ module GoodData
     #
     # * +username+ - The GoodData account username
     # * +password+ - The GoodData account password
-    def initialize(username, password, url = nil)
+    def initialize(username, password, url = nil, options = {})
       @status   = :not_connected
       @username = username
       @password = password
       @url      = url || DEFAULT_URL
+      @options  = options
     end
 
     # Returns the user JSON object of the currently logged in GoodData user account.
@@ -165,11 +166,13 @@ module GoodData
         }
       }
 
-      @server = RestClient::Resource.new @url, :headers => {
-        :content_type => :json,
-        :accept => [ :json, :zip ],
-        :user_agent => GoodData.gem_version_string
-      }
+      @server = RestClient::Resource.new @url,
+        :timeout => @options[:timeout],
+        :headers => {
+          :content_type => :json,
+          :accept => [ :json, :zip ],
+          :user_agent => GoodData.gem_version_string,
+        }
 
       GoodData.logger.debug "Logging in..."
       @user = post(LOGIN_PATH, credentials, :dont_reauth => true)['userLogin']

data/lib/gooddata/extract.rb

@@ -1,6 +1,6 @@
 require 'csv'
 
-module GoodData::Extract  
+module GoodData::Extract
   class CsvFile
     def initialize(file)
       @file = file

data/lib/gooddata/model.rb

@@ -6,7 +6,9 @@ require 'fastercsv'
 # elements, including the server-side data model.
 #
 module GoodData
+
   module Model
+
     # GoodData REST API categories
     LDM_CTG = 'ldm'
     LDM_MANAGE_CTG = 'ldm-manage'
@@ -20,12 +22,19 @@ module GoodData
     LABEL_COLUMN_PREFIX = 'nm_'
     ATTRIBUTE_FOLDER_PREFIX = 'dim'
     ATTRIBUTE_PREFIX = 'attr'
+    LABEL_PREFIX = 'label'
     FACT_PREFIX = 'fact'
     DATE_FACT_PREFIX = 'dt'
+    DATE_ATTRIBUTE = "date"
+    DATE_ATTRIBUTE_DEFAULT_DISPLAY_FORM = 'mdyy'
     TIME_FACT_PREFIX = 'tm.dt'
     TIME_ATTRIBUTE_PREFIX = 'attr.time'
     FACT_FOLDER_PREFIX = 'ffld'
 
+    SKIP_FIELD = false
+
+    BEGINNING_OF_TIMES = Date.parse('1/1/1900')
+
     class << self
       def add_dataset(title, columns, project = nil)
         add_schema Schema.new('columns' => columns, 'title' => title), project
@@ -74,7 +83,7 @@ module GoodData
     # model abstractions.
     #
     class Schema < MdObject
-      attr_reader :fields
+      attr_reader :fields, :attributes, :facts, :folders, :references, :labels
 
       def self.load(file)
         Schema.new JSON.load(open(file))
@@ -82,20 +91,57 @@ module GoodData
 
       def initialize(config, title = nil)
         @fields = []
+        @attributes = {}
+        @facts = {}
+        @folders = {
+          :facts      => {},
+          :attributes => {}
+        }
+        @references = {}
+        @labels = []
+
         config['title'] = title unless config['title']
         raise 'Schema name not specified' unless config['title']
         self.title = config['title']
         self.config = config
       end
 
+      def transform_header(headers)
+        result = fields.reduce([]) do |memo, f|
+          val = f.to_csv_header(headers)
+          memo << val unless val === SKIP_FIELD
+          memo
+        end
+        result.flatten
+      end
+
+      def transform_row(headers, row)
+        result = fields.reduce([]) do |memo, f|
+          val = f.to_csv_data(headers, row)
+          memo << val unless val === SKIP_FIELD
+          memo
+        end
+        result.flatten
+      end
+
       def config=(config)
-        labels = []
         config['columns'].each do |c|
-          add_attribute c if c['type'] == 'ATTRIBUTE'
-          add_fact c if c['type'] == 'FACT'
-          add_date c if c['type'] == 'DATE'
-          set_connection_point c if c['type'] == 'CONNECTION_POINT'
-          labels.push c if c['type'] == 'LABEL'
+          case c['type']
+          when 'ATTRIBUTE'
+            add_attribute c
+          when 'FACT'
+            add_fact c
+          when 'DATE'
+            add_date c
+          when 'CONNECTION_POINT'
+            set_connection_point c
+          when 'LABEL'
+            add_label c
+          when 'REFERENCE'
+            add_reference c
+          else
+            fail "Unexpected type #{c['type']} in #{c.inspect}"
+          end
         end
         @connection_point = RecordsOf.new(nil, self) unless @connection_point
       end
@@ -107,10 +153,6 @@ module GoodData
 
       def type_prefix ; 'dataset' ; end
 
-      def attributes; @attributes ||= {} ; end
-      def facts; @facts ||= {} ; end
-      def folders; @folders ||= {}; end
-
       ##
       # Underlying fact table name
       #
@@ -142,24 +184,43 @@ module GoodData
             maql += "ALTER DATASET {#{self.identifier}} ADD {#{obj.identifier}};\n\n"
           end
         end
+
+        labels.each do |label|
+          maql += "# Creating Labels\n"
+          maql += label.to_maql_create
+        end
+
+        references.values.each do |ref|
+          maql += "# Creating references\n"
+          maql += ref.to_maql_create
+        end
+
         folders_maql = "# Create folders\n"
-        folders.keys.each { |folder| folders_maql += folder.to_maql_create }
+        (folders[:attributes].values + folders[:facts].values).each { |folder| folders_maql += folder.to_maql_create }
         folders_maql + "\n" + maql + "SYNCHRONIZE {#{identifier}};\n"
       end
 
       # Load given file into a data set described by the given schema
       #
-      def upload(path, project = nil)
+      def upload(path, project = nil, mode = "FULL")
         path = path.path if path.respond_to? :path
+        header = nil
         project = GoodData.project unless project
 
         # create a temporary zip file
         dir = Dir.mktmpdir
         Zip::ZipFile.open("#{dir}/upload.zip", Zip::ZipFile::CREATE) do |zip|
           # TODO make sure schema columns match CSV column names
-          zip.get_output_stream('upload_info.json') { |f| f.puts JSON.pretty_generate(to_manifest) }
+          zip.get_output_stream('upload_info.json') { |f| f.puts JSON.pretty_generate(to_manifest(mode)) }
           zip.get_output_stream('data.csv') do |f|
-            FasterCSV.foreach(path) { |row| f.puts row.to_csv }
+            FasterCSV.foreach(path, :headers => true, :return_headers => true) do |row|
+              output = if row.header_row?
+                transform_header(row)
+              else
+                transform_row(header, row)
+              end
+              f.puts output.to_csv
+            end
           end
         end
 
@@ -170,18 +231,22 @@ module GoodData
         # kick the load
         pull = { 'pullIntegration' => File.basename(dir) }
         link = project.md.links('etl')['pull']
-        GoodData.post link, pull
+        task = GoodData.post link, pull
+        while (GoodData.get(task["pullTask"]["uri"])["taskStatus"] === "RUNNING" || GoodData.get(task["pullTask"]["uri"])["taskStatus"] === "PREPARED") do
+          sleep 30
+        end
+        puts "Done loading"
       end
 
       # Generates the SLI manifest describing the data loading
-      #
-      def to_manifest
+      # 
+      def to_manifest(mode)
         {
           'dataSetSLIManifest' => {
-            'parts'   => fields.map { |f| f.to_manifest_part },
+            'parts'   => fields.reduce([]) { |memo, f| val = f.to_manifest_part(mode); memo << val unless val.nil?; memo },
             'dataSet' => self.identifier,
             'file'    => 'data.csv', # should be configurable
-            'csvParams' => {
+              'csvParams' => {
               'quoteChar'     => '"',
               'escapeChar'    => '"',
               'separatorChar' => ',',
@@ -195,24 +260,51 @@ module GoodData
 
       def add_attribute(column)
         attribute = Attribute.new column, self
-        @fields << attribute
-        add_to_hash(self.attributes, attribute)
-        folders[AttributeFolder.new(attribute.folder)] = 1 if attribute.folder
+        fields << attribute
+        add_to_hash(attributes, attribute)
+        add_attribute_folder(attribute.folder)
+        # folders[AttributeFolder.new(attribute.folder)] = 1 if attribute.folder
+      end
+
+      def add_attribute_folder(name)
+        return if name.nil?
+        return if folders[:attributes].has_key?(name)
+        folders[:attributes][name] = AttributeFolder.new(name)
       end
 
       def add_fact(column)
         fact = Fact.new column, self
-        @fields << fact
-        add_to_hash(self.facts, fact)
-        folders[FactFolder.new(fact.folder)] = 1 if fact.folder
+        fields << fact
+        add_to_hash(facts, fact)
+        add_fact_folder(fact.folder)
+        # folders[FactFolder.new(fact.folder)] = 1 if fact.folder
+      end
+
+      def add_fact_folder(name)
+        return if name.nil?
+        return if folders[:facts].has_key?(name)
+        folders[:facts][name] = FactFolder.new(name)
+      end
+
+      def add_label(column)
+        label = Label.new(column, nil, self)
+        labels << label
+        fields << label
+      end
+
+      def add_reference(column)
+        reference = Reference.new(column, self)
+        fields << reference
+        add_to_hash(references, reference)
       end
 
       def add_date(column)
         date = DateColumn.new column, self
+        @fields << date
         date.parts.values.each { |p| @fields << p }
         date.facts.each { |f| add_to_hash(self.facts, f) }
         date.attributes.each { |a| add_to_hash(self.attributes, a) }
-        @fields << date
+        date.references.each {|r| add_to_hash(self.references, r)}
       end
 
       def set_connection_point(column)
@@ -256,6 +348,15 @@ module GoodData
         visual
       end
 
+      def to_csv_header(row)
+        name
+      end
+
+      def to_csv_data(headers, row)
+        row[name]
+      end
+
+
       # Overriden to prevent long strings caused by the @schema attribute
       #
       def inspect
@@ -286,16 +387,17 @@ module GoodData
       def key ; "#{Model::to_id(@name)}#{FK_SUFFIX}" ; end
 
       def to_maql_create
-        "CREATE ATTRIBUTE {#{identifier}} VISUAL (#{visual})" \
-               + " AS KEYS {#{table}.#{Model::FIELD_PK}} FULLSET;\n" \
-               + @primary_label.to_maql_create
+        maql = "CREATE ATTRIBUTE {#{identifier}} VISUAL (#{visual})" \
+               + " AS KEYS {#{table}.#{Model::FIELD_PK}} FULLSET;\n"
+        maql += @primary_label.to_maql_create if @primary_label
+        maql
       end
 
-      def to_manifest_part
+      def to_manifest_part(mode)
         {
           'referenceKey' => 1,
           'populates' => [ @primary_label.identifier ],
-          'mode' => 'FULL',
+          'mode' => mode,
           'columnName' => name
         }
       end
@@ -309,21 +411,23 @@ module GoodData
     class Label < Column
       def type_prefix ; 'label' ; end
 
+      # def initialize(hash, schema)
       def initialize(hash, attribute, schema)
         super hash, schema
-        @attribute = attribute
+        @attribute = attribute || schema.fields.find {|field| field.name === hash["reference"]}
       end
 
       def to_maql_create
+        "# LABEL FROM LABEL"
         "ALTER ATTRIBUTE {#{@attribute.identifier}} ADD LABELS {#{identifier}}" \
               + " VISUAL (TITLE #{title.inspect}) AS {#{column}};\n"
       end
 
-      def to_manifest_part
+      def to_manifest_part(mode)
         {
-          'populates'  => [ identifier ],
-          'mode'       => 'FULL',
-          'columnName' => name
+          'populates'     => [ identifier ],
+          'mode'          => mode,
+          'columnName'    => name
         }
       end
 
@@ -389,11 +493,11 @@ module GoodData
                + " AS {#{column}};\n"
       end
 
-      def to_manifest_part
+      def to_manifest_part(mode)
         {
           'populates'  => [ identifier ],
-          'mode'       => 'FULL',
-          'columnName' => column
+          'mode'       => mode,
+          'columnName' => name
         }
       end
     end
@@ -401,42 +505,46 @@ module GoodData
     ##
     # Reference to another data set
     #
-    class Reference
+    class Reference < Column
       def initialize(column, schema)
+        super column, schema
+        # pp column
+
         @name       = column['name']
         @reference  = column['reference']
-        @schema_ref = column['schema_ref']
+        @schema_ref = column['schema_reference']
         @schema     = schema
       end
 
       ##
-      # Generates an identifier of the referencing attribute using the 
+      # Generates an identifier of the referencing attribute using the
       # schema name derived from schemaReference and column name derived
       # from the reference key.
       #
       def identifier
-        @identifier ||= "#{ATTRIBUTE_PREFIX}.#{Model::to_id @schema_ref.title}.#{Model::to_id @reference}"
+        @identifier ||= "#{ATTRIBUTE_PREFIX}.#{Model::to_id @schema_ref}.#{Model::to_id @reference}"
       end
 
       def key ; "#{Model::to_id @name}_id" ; end
 
       def label_column
-        @column ||= "#{@schema.table}.#{LABEL_COLUMN_PREFIX Model::to_id(reference)}"
+        "#{LABEL_PREFIX}.#{Model::to_id @schema_ref}.#{Model::to_id @reference}"
       end
 
       def to_maql_create
-        "ALTER ATTRIBUTE {#{self.identifier} ADD KEYS {#{@schema.table}.#{key}}"
+        "ALTER ATTRIBUTE {#{self.identifier}} ADD KEYS {#{@schema.table}.#{key}};\n"
       end
 
       def to_maql_drop
-        "ALTER ATTRIBUTE {#{self.identifier} DROP KEYS {#{@schema.table}.#{key}}"
+        "ALTER ATTRIBUTE {#{self.identifier} DROP KEYS {#{@schema.table}.#{key}};\n"
       end
 
-      def to_manifest_part
+      def to_manifest_part(mode)
         {
-          'populates'  => [ identifier ],
-          'mode'       => 'FULL',
-          'columnName' => label_column
+          'populates'     => [ label_column ],
+          'mode'          => mode,
+          'columnName'    => name,
+          'referenceKey'  => 1
         }
       end
     end
@@ -445,8 +553,35 @@ module GoodData
     # Fact representation of a date.
     #
     class DateFact < Fact
+
+      attr_accessor :format, :output_format
+
+      def initialize(column, schema)
+        super column, schema
+        @output_format = column["format"] || '("dd/MM/yyyy")'
+        @format = @output_format.gsub('yyyy', '%Y').gsub('MM', '%m').gsub('dd', '%d')
+      end
+
       def column_prefix ; DATE_COLUMN_PREFIX ; end
       def type_prefix ; DATE_FACT_PREFIX ; end
+
+      def to_csv_header(row)
+        "#{name}_fact"
+      end
+
+      def to_csv_data(headers, row)
+        val = row[name]
+        val.nil?() ? nil : (Date.strptime(val, format) - BEGINNING_OF_TIMES).to_i
+      end
+
+      def to_manifest_part(mode)
+        {
+          'populates'  => [ identifier ],
+          'mode'       => mode,
+          'columnName' => "#{name}_fact"
+        }
+      end
+
     end
 
     ##
@@ -454,6 +589,38 @@ module GoodData
     #
     class DateReference < Reference
 
+      attr_accessor :format, :output_format, :urn
+
+      def initialize(column, schema)
+        super column, schema
+        @output_format = column["format"] || '("dd/MM/yyyy")'
+        @format = @output_format.gsub('yyyy', '%Y').gsub('MM', '%m').gsub('dd', '%d')
+        @urn = column["urn"] || "URN:GOODDATA:DATE"
+      end
+
+      def identifier
+        @identifier ||= "#{Model::to_id @schema_ref}.#{DATE_ATTRIBUTE}"
+      end
+
+      def to_manifest_part(mode)
+        {
+          'populates'     => [ "#{identifier}.#{DATE_ATTRIBUTE_DEFAULT_DISPLAY_FORM}" ],
+          'mode'          => mode,
+          'constraints'   => {"date" => output_format},
+          'columnName'    => name,
+          'referenceKey'  => 1
+        }
+      end
+
+      def to_maql_create
+        # urn:chefs_warehouse_fiscal:date
+        super_maql = super
+        maql = ""
+        # maql = "# Include date dimensions\n"
+        # maql += "INCLUDE TEMPLATE \"#{urn}\" MODIFY (IDENTIFIER \"#{name}\", TITLE \"#{title || name}\");\n"
+        maql += super_maql
+      end
+
     end
 
     ##
@@ -461,6 +628,15 @@ module GoodData
     #
     class DateAttribute < Attribute
       def key ; "#{DATE_COLUMN_PREFIX}#{super}" ; end
+
+      def to_manifest_part(mode)
+        {
+          'populates'     => ['label.stuff.mmddyy'],
+          "format"        => "unknown",
+          "mode"          => mode,
+          "referenceKey"  => 1
+        }
+      end
     end
 
     ##
@@ -492,22 +668,24 @@ module GoodData
     # parts: date fact, a date reference or attribute and an optional time component
     # that contains a time fact and a time reference or attribute.
     #
-    class DateColumn
-      attr_reader :parts, :facts, :attributes
+    class DateColumn < Column
+      attr_reader :parts, :facts, :attributes, :references
 
       def initialize(column, schema)
-        @parts = {} ; @facts = [] ; @attributes = []
+        super column, schema
+        @parts = {} ; @facts = [] ; @attributes = []; @references = []
 
         @facts << @parts[:date_fact] = DateFact.new(column, schema)
-        if column['schemaReference'] then
+        if column['schema_reference'] then
           @parts[:date_ref] = DateReference.new column, schema
+          @references << @parts[:date_ref]
         else
           @attributes << @parts[:date_attr] = DateAttribute.new(column, schema)
         end
         if column['datetime'] then
           puts "*** datetime"
           @facts << @parts[:time_fact] = TimeFact.new(column, schema)
-          if column['schemaReference'] then
+          if column['schema_reference'] then
             @parts[:time_ref] = TimeReference.new column, schema
           else
             @attributes << @parts[:time_attr] = TimeAttribute.new(column, schema)
@@ -522,6 +700,19 @@ module GoodData
       def to_maql_drop
         @parts.values.map { |v| v.to_maql_drop }.join "\n"
       end
+
+      def to_csv_header(row)
+        SKIP_FIELD
+      end
+
+      def to_csv_data(headers, row)
+        SKIP_FIELD
+      end
+
+      def to_manifest_part(mode)
+        nil
+      end
+
     end
 
     ##
@@ -554,5 +745,16 @@ module GoodData
       def type; "FACT"; end
       def type_prefix; "ffld"; end
     end
+
+    class DateDimension < MdObject
+
+      def to_maql_create
+        # urn:chefs_warehouse_fiscal:date
+        maql = ""
+        maql += "INCLUDE TEMPLATE \"#{urn}\" MODIFY (IDENTIFIER \"#{name}\", TITLE \"#{title || name}\");"
+        maql
+      end
+    end
+
   end
 end

data/lib/gooddata/models/data_result.rb

@@ -0,0 +1,241 @@
+require 'fastercsv'
+
+module GoodData
+
+  class Row < FasterCSV::Row
+    def ==(other)
+       len = length()
+       return false if len != other.length
+       result = true
+
+       len.times do |i|
+         result = false unless convert_field(field(i)) == convert_field(other.field(i))
+       end
+       result
+    end
+
+    private
+    def convert_field(val)
+      if val.is_a?(String) && val.match(/^[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?$/)
+        val = val.scan(/[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?/).first
+        val = val.include?('.') ? val.to_f.round : val.to_i
+        return val
+      elsif val.nil? || val == ' '
+        return 'N/A'
+      elsif val.respond_to? :round
+        return val.round
+      else
+        return val
+      end
+    end
+  end
+
+  class DataResult
+
+    attr_reader :data
+
+    def initialize(data)
+      @data = data
+    end
+
+    def print
+      a = to_table.to_a
+      a.transpose.unshift((1..a.length).to_a).each_with_index.map{|col, i|
+        col.unshift(i.zero?? nil : i)   # inserts row labels #
+        w = col.map{|cell| cell.to_s.length}.max   # w = "column width" #
+        col.each_with_index.map{|cell, i|
+          i.zero?? cell.to_s.center(w) : cell.to_s.ljust(w)}   # alligns the column #
+      }.transpose.each{|row| puts "[#{row.join(' | ')}]"}
+    end
+
+    def to_table
+      raise "Should be implemented in subclass"
+    end
+
+  end
+
+
+  class SFDataResult < DataResult
+
+    def initialize(data, options = {})
+      super(data)
+      @options = options
+      assemble_table
+    end
+
+    def assemble_table
+      sf_data = data[:queryResponse][:result][:records]
+      sf_data = sf_data.is_a?(Hash) ? [sf_data] : sf_data
+      if @options[:soql]
+        @headers = @options[:soql].strip.match(/^SELECT (.*) FROM/)[1].strip.split(",").map{|item| item.strip.split(/\s/)}.map{|item| item.last.to_sym}
+      elsif @options[:headers]
+        @headers = @options[:headers]
+      else
+        @headers = sf_data.first.keys - [:type, :Id]
+      end
+      @table = FasterCSV::Table.new(sf_data.collect do |line|
+        GoodData::Row.new([], @headers.map {|h| line[h] || ' '}, false)
+      end)
+    rescue
+      fail "Unable to assemble the table. Either the data provided are empty or the SOQL is malformed."
+    end
+
+    def to_table
+      @table
+    end
+
+    def == (otherDataResult)
+      result = true
+      len =  @table.length
+      other_table = otherDataResult.to_table
+      if len != other_table.length
+        # puts "TABLES ARE OF DIFFERENT SIZES"
+        return false
+      end
+
+      diff(otherDataResult).empty?() ? true : false
+
+    end
+
+    def diff(otherDataResult)
+      other_table = otherDataResult.to_table
+      differences = []
+
+      @table.each do |row|
+        differences << row unless other_table.detect {|r| r == row}
+      end
+      differences
+    end
+
+  end
+
+  class ReportDataResult < DataResult
+
+    attr_reader :row_headers, :column_headers, :table, :headers_height, :headers_width
+
+    def initialize(data)
+      super
+      @row_headers      = []
+      @column_headers   = []
+      @table            = []
+
+      @row_headers, @headers_width = tabularize_rows
+      @column_headers, @headers_height = tabularize_columns
+
+      assemble_table
+    end
+
+    def without_column_headers
+      @table = table.transpose[headers_height, 1000000].transpose
+      self
+    end
+
+    def each_line
+      table.transpose.each {|line| yield line}
+    end
+
+    def to_table
+      FasterCSV::Table.new(table.transpose.map {|line| GoodData::Row.new([], line.map {|item| item || ' '}, false)})
+    end
+
+    def == (otherDataResult)
+      result = true
+      csv_table = to_table
+      len =  csv_table.length
+      return false if len != otherDataResult.to_table.length
+
+
+      result
+    end
+
+    def diff(otherDataResult)
+      csv_table = to_table
+      other_table = otherDataResult.to_table
+      differences = []
+
+      csv_table.each do |row|
+        differences << row unless other_table.detect {|r| r == row}
+      end
+      differences
+    end
+
+    private
+    def each_level(table, level, children, lookup)
+      max_level = level + 1
+      children.each do |kid|
+        first = kid["first"]
+        last = kid["last"]
+        repetition = last - first + 1
+        repetition.times do |i|
+          table[first + i] ||= []
+          if kid["type"] == 'total'
+            table[first + i][level] = kid["id"]
+          else
+            table[first + i][level] = lookup[level][kid["id"].to_s]
+          end
+        end
+        if (!kid["children"].empty?)
+          new_level = each_level(table, level+1, kid["children"], lookup)
+          max_level = [max_level, new_level].max
+        end
+      end
+      max_level
+    end
+
+    def tabularize_rows
+      rows    = data["xtab_data"]["rows"]
+      kids = rows["tree"]["children"]
+
+      if kids.empty? || (kids.size == 1 && kids.first['type'] == 'metric')
+        headers, size = [[nil]], 0
+      else
+        headers = []
+        size = each_level(headers, 0, rows["tree"]["children"], rows["lookups"])
+      end
+      return headers, size
+    end
+
+    def tabularize_columns
+      columns = data["xtab_data"]["columns"]
+      kids = columns["tree"]["children"]
+
+      if kids.empty? || (kids.size == 1 && kids.first['type'] == 'metric')
+        headers, size = [[nil]], 0
+      else
+        headers = []
+        size = each_level(headers, 0, columns["tree"]["children"], columns["lookups"])
+      end
+      return headers, size
+    end
+
+    def assemble_table()
+  #    puts "=== COLUMNS === #{column_headers.size}x#{headers_height}"
+      (column_headers.size).times do |i|
+        (headers_height).times do |j|
+          table[headers_width + i] ||= []
+  #        puts "[#{headers_width + i}][#{j}] #{column_headers[i][j]}"
+          table[headers_width + i][j] = column_headers[i][j]
+        end
+      end
+
+  #    puts "=== ROWS ==="
+      (row_headers.size).times do |i|
+        (headers_width).times do |j|
+          table[j] ||= []
+  #        puts "[#{j}][#{headers_height + i}] #{row_headers[i][j]}"
+          table[j][headers_height + i] = row_headers[i][j]
+        end
+      end
+
+      xtab_data = data["xtab_data"]["data"]
+  #    puts "=== DATA === #{column_headers.size}x#{row_headers.size}"
+      (column_headers.size).times do |i|
+        (row_headers.size).times do |j|
+          table[headers_width + i] ||= []
+  #        puts "[#{headers_width + i}, #{headers_height + j}] [#{i}][#{j}]=#{xtab_data[j][i]}"
+          table[headers_width + i][headers_height + j] = xtab_data[j][i]
+        end
+      end
+    end
+  end
+end

data/lib/gooddata/models/metadata.rb

@@ -39,6 +39,14 @@ module GoodData
       GoodData.delete @json['links']['self']
     end
 
+    def obj_id
+      uri.split('/').last
+    end
+
+    def links
+      @json['links']
+    end
+
     def uri
       meta['uri']
     end
@@ -51,6 +59,10 @@ module GoodData
       meta['title']
     end
 
+    def summary
+      meta['summary']
+    end
+
     def meta
       @json['meta']
     end

data/lib/gooddata/models/project.rb

@@ -87,6 +87,10 @@ module GoodData
       @json['links']['self'] if @json['links'] && @json['links']['self']
     end
 
+    def obj_id
+      uri.split('/').last
+    end
+
     def title
       @json['meta']['title'] if @json['meta']
     end
@@ -112,8 +116,8 @@ module GoodData
       Model.add_schema schema, self
     end
 
-    def upload(file, schema)
-      schema.upload file, self
+    def upload(file, schema, mode = "FULL")
+      schema.upload file, self, mode
     end
 
     def slis

data/lib/gooddata/models/report.rb

@@ -0,0 +1,29 @@
+module GoodData
+  class Report < GoodData::MdObject 
+
+    class << self
+      def [](id)
+        if id == :all
+          GoodData.get(GoodData.project.md['query'] + '/reports/')['query']['entries']
+        else 
+          super
+        end
+      end
+    end
+
+    def execute
+      # puts "Executing report #{uri}"
+      result = GoodData.post '/gdc/xtab2/executor3', {"report_req" => {"report" => uri}}
+      dataResultUri = result["reportResult2"]["content"]["dataResult"]
+
+      result = GoodData.get dataResultUri
+      while result["taskState"] && result["taskState"]["state"] == "WAIT" do
+        sleep 10
+        result = GoodData.get dataResultUri
+      end
+      data_result = ReportDataResult.new(GoodData.get dataResultUri)
+      data_result
+    end
+
+  end
+end

data/lib/gooddata/version.rb

@@ -1,3 +1,3 @@
 module GoodData
-  VERSION = "0.1.0"
+  VERSION = "0.4.0"
 end

data/test/helper.rb

@@ -6,5 +6,8 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'gooddata'
 
+# used in test that expect an existing accessible project
+$DEMO_PROJECT = 'ca6a1r1lbfwpt2v05k36nbc0cjpu7lh9'
+
 class Test::Unit::TestCase
 end

data/test/test_commands.rb

@@ -32,7 +32,7 @@ class TestRestApiBasic < Test::Unit::TestCase
     }
 
     should "list datasets" do
-      GoodData::Command.run "datasets", [ "--project", "FoodMartDemo" ]
+      GoodData::Command.run "datasets", [ "--project", $DEMO_PROJECT ]
     end
 
     should "apply a dataset model" do
@@ -60,8 +60,8 @@ class TestRestApiBasic < Test::Unit::TestCase
       GoodData::Command.run "api:test", []
     end
 
-    should "get FoodMartDemo metadata" do
-      GoodData::Command.run "api:get", [ '/gdc/md/FoodMartDemo' ]
+    should "get demo project's metadata" do
+      GoodData::Command.run "api:get", [ "/gdc/md/#{$DEMO_PROJECT}" ]
     end
   end
 

data/test/test_model.rb

@@ -45,6 +45,23 @@ class TestModel < Test::Unit::TestCase
       ds = GoodData::DataSet['dataset.mrkev']
       assert_not_nil ds
 
+      project.delete
+    end
+
+    should "create a simple model with no CP in a sandbox project using Model.add_dataset" do
+      project = GoodData::Project.create :title => "gooddata-ruby test #{Time.new.to_i}"
+      GoodData.use project
+
+      # create a similar data set but without the connection point column
+      cols_no_cp = COLUMNS.select { |c| c['type'] != 'CONNECTION_POINT' }
+      objects    = GoodData::Model.add_dataset 'No CP', cols_no_cp
+      uris       = objects['uris']
+
+      # Repeat check of metadata objects expected to be created on the server side
+      GoodData.get uris[uris.length - 1]
+      ds = GoodData::DataSet['dataset.nocp']
+      assert_not_nil ds
+
       # clean-up
       project.delete
     end

data/test/test_rest_api_basic.rb

@@ -13,29 +13,29 @@ class TestRestApiBasic < Test::Unit::TestCase
       GoodData::Command::connect
     end
 
-    should "get the FoodMartDemo" do
-      p_by_hash   = GoodData::Project['FoodMartDemo']
-      p_by_uri    = GoodData::Project['/gdc/projects/FoodMartDemo']
-      p_by_md_uri = GoodData::Project['/gdc/md/FoodMartDemo']
+    should "get the demo project" do
+      p_by_hash   = GoodData::Project[$DEMO_PROJECT]
+      p_by_uri    = GoodData::Project["/gdc/projects/#{$DEMO_PROJECT}"]
+      p_by_md_uri = GoodData::Project["/gdc/md/#{$DEMO_PROJECT}"]
       assert_not_nil p_by_hash
       assert_equal p_by_hash.uri, p_by_uri.uri
       assert_equal p_by_hash.title, p_by_uri.title
       assert_equal p_by_hash.title, p_by_md_uri.title
     end
 
-    should "connect to the FoodMartDemo" do
-      GoodData.use 'FoodMartDemo'
+    should "connect to the demo project" do
+      GoodData.use $DEMO_PROJECT
       GoodData.project.datasets # should not fail on unknown project or access denied
                                 # TODO: should be equal to Dataset.all once implemented
     end
 
     # Not supported yet
     # should "fetch dataset by numerical or string identifier" do
-    #   GoodData.use 'FoodMartDemo'
+    #   GoodData.use $DEMO_PROJECT
     #   ds_by_hash = Dataset['amJoIYHjgESv']
     #   ds_by_id   = Dataset[34]
     #   assert_not_nil ds_by_hash
     #   assert_equal ds_by_hash.uri, ds_by_id.uri
     # end
   end
-end
+end

data/test/test_upload.rb

@@ -46,7 +46,9 @@ class TestModel < Test::Unit::TestCase
 
     should "upload CSV in a full mode" do
       @project.add_dataset SCHEMA
-      @project.upload @file.path, SCHEMA
+      assert_equal 1, @project.datasets.size
+      assert_equal "test", @project.datasets.first.title
+      @project.upload @file.path, SCHEMA, "FULL"
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: gooddata
 version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: false
+  hash: 15
+  prerelease: 
   segments: 
   - 0
-  - 2
+  - 4
   - 0
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Pavel Kolesnikov
@@ -16,12 +16,9 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-10 00:00:00 -07:00
-default_executable: 
+date: 2011-09-20 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: thoughtbot-shoulda
-  prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -31,11 +28,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :development
   version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: parseconfig
+  name: thoughtbot-shoulda
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency 
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -45,11 +42,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :runtime
   version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: json_pure
+  name: parseconfig
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency 
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -59,11 +56,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :runtime
   version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: rest-client
+  name: json_pure
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency 
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -73,11 +70,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :runtime
   version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: fastercsv
+  name: rest-client
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency 
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -87,11 +84,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :runtime
   version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: json
+  name: fastercsv
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency 
   requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -101,11 +98,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :runtime
   version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  name: rubyzip
+  name: json
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency 
   requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -115,13 +112,15 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :runtime
   version_requirements: *id007
+  name: rubyzip
+  prerelease: false
+  type: :runtime
 description: Use the Gooddata::Client class to integrate GoodData into your own application or use the CLI to work with GoodData directly from the command line.
 email: pavel@gooddata.com
 executables: 
-- gooddata
 - igd.rb
+- gooddata
 extensions: []
 
 extra_rdoc_files: 
@@ -148,10 +147,12 @@ files:
 - lib/gooddata/extract.rb
 - lib/gooddata/helpers.rb
 - lib/gooddata/model.rb
+- lib/gooddata/models/data_result.rb
 - lib/gooddata/models/links.rb
 - lib/gooddata/models/metadata.rb
 - lib/gooddata/models/profile.rb
 - lib/gooddata/models/project.rb
+- lib/gooddata/models/report.rb
 - lib/gooddata/version.rb
 - test/helper.rb
 - test/test_commands.rb
@@ -159,13 +160,12 @@ files:
 - test/test_model.rb
 - test/test_rest_api_basic.rb
 - test/test_upload.rb
-has_rdoc: true
 homepage: http://github.com/gooddata/gooddata-ruby
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -189,14 +189,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: A convenient Ruby wrapper around the GoodData RESTful API
-test_files: 
-- test/helper.rb
-- test/test_commands.rb
-- test/test_guessing.rb
-- test/test_model.rb
-- test/test_rest_api_basic.rb
-- test/test_upload.rb
+test_files: []
+