gooddata 0.6.0.pre6 → 0.6.0.pre7

data/lib/gooddata/helpers.rb

@@ -1,18 +1,45 @@
 module GoodData::Helpers
-  def home_directory
+  def self.home_directory
     running_on_windows? ? ENV['USERPROFILE'] : ENV['HOME']
   end
 
-  def running_on_windows?
+  def self.running_on_windows?
     RUBY_PLATFORM =~ /mswin32|mingw32/
   end
 
-  def running_on_a_mac?
+  def self.running_on_a_mac?
     RUBY_PLATFORM =~ /-darwin\d/
   end
 
-  def error(msg)
+  def self.error(msg)
     STDERR.puts(msg)
     exit 1
   end
+
+  def self.find_goodfile(pwd, options={})
+    root = Pathname(options[:root] || '/' )
+    pwd = Pathname(pwd).expand_path
+    begin
+       gf = pwd + "Goodfile"
+       if gf.exist?
+         return gf
+       end
+       pwd = pwd.parent
+    end until root == pwd
+    fail "Goodfile not found in #{pwd.to_s} or any parent up to #{root.to_s}"
+  end
+
+  def self.hash_dfs(thing, &block)
+    if !thing.is_a?(Hash) && !thing.is_a?(Array)
+    elsif thing.is_a?(Array)
+      thing.each do |child|
+        hash_dfs(child, &block)
+      end
+    else
+      thing.each do |key, val|
+        yield(thing, key)
+        hash_dfs(val, &block)
+      end
+    end
+  end
 end

data/lib/gooddata/model.rb

@@ -1,5 +1,5 @@
 require 'open-uri'
-
+require 'active_support/all'
 ##
 # Module containing classes that counter-part GoodData server-side meta-data
 # elements, including the server-side data model.
@@ -49,6 +49,134 @@ module GoodData
         GoodData.post ldm_uri, { 'manage' => { 'maql' => schema.to_maql_create } }
       end
 
+      # Load given file into a data set described by the given schema
+      def upload_data(path, manifest, options={})
+        project = options[:project] || GoodData.project
+        # mode = options[:mode] || "FULL"
+        path = path.path if path.respond_to? :path
+        inline_data = path.is_a?(String) ? false : true
+
+        # create a temporary zip file
+        dir = Dir.mktmpdir
+        begin
+          Zip::File.open("#{dir}/upload.zip", Zip::File::CREATE) do |zip|
+            # TODO make sure schema columns match CSV column names
+            zip.get_output_stream('upload_info.json') { |f| f.puts JSON.pretty_generate(manifest) }
+            if inline_data
+              zip.get_output_stream('data.csv') do |f|
+                path.each do |row|
+                  f.puts row.to_csv
+                end
+              end
+            else
+              zip.add('data.csv', path)
+            end
+          end
+
+          # upload it
+          GoodData.upload_to_user_webdav("#{dir}/upload.zip", :directory => File.basename(dir))
+        ensure
+          FileUtils.rm_rf dir
+        end
+
+        # kick the load
+        pull = { 'pullIntegration' => File.basename(dir) }
+        link = project.md.links('etl')['pull']
+        task = GoodData.post link, pull
+        while (GoodData.get(task["pullTask"]["uri"])["taskStatus"] === "RUNNING" || GoodData.get(task["pullTask"]["uri"])["taskStatus"] === "PREPARED") do
+          sleep 30
+        end
+        if (GoodData.get(task["pullTask"]["uri"])["taskStatus"] == "ERROR")
+          s = StringIO.new
+          GoodData.download_form_user_webdav(File.basename(dir) + '/upload_status.json', s)
+          js = JSON.parse(s.string)
+          fail "Load Failed with error #{JSON.pretty_generate(js)}"
+        end
+        puts "Done loading"
+      end
+
+    end
+
+    class ProjectBlueprint
+
+      attr_accessor :data
+
+      def change(&block)
+        builder = ProjectBuilder.create_from_data(self)
+        block.call(builder)
+        builder
+        @data = builder.to_hash
+        self
+      end
+
+      def datasets
+        data[:datasets].map {|d| SchemaBlueprint.new(d)}
+      end
+
+      def get_dataset(name)
+        ds = data[:datasets].find {|d| d[:name] == name}
+        SchemaBlueprint.new(ds) unless ds.nil?
+      end
+
+      def initialize(init_data)
+        @data = init_data
+      end
+
+    end
+
+    class SchemaBlueprint
+
+      attr_accessor :data
+
+      def change(&block)
+        builder = SchemaBuilder.create_from_data(self)
+        block.call(builder)
+        builder
+        @data = builder.to_hash
+        self
+      end
+
+      def initialize(init_data)
+        @data = init_data
+      end
+
+      def upload(source, options={})
+        project = options[:project] || GoodData.project
+        mode = options[:load] || "FULL"
+        project.upload(source, to_schema, mode)
+      end
+
+      def name
+        data[:name]
+      end
+
+      def title
+        data[:title]
+      end
+
+      def to_hash
+        data
+      end
+
+      def columns
+        data[:columns]
+      end
+
+      def to_schema
+        Schema.new(to_hash)
+      end
+
+      def to_manifest
+        to_schema.to_manifest
+      end
+      
+      def pretty_print(printer)
+        printer.text "Schema <#{object_id}>:\n" 
+        printer.text " Name: #{name}\n"
+        printer.text " Columns: \n"
+        printer.text columns.map {|c| "  #{c[:name]}: #{c[:type]}"}.join("\n")
+      end
+
     end
 
     class ProjectBuilder
@@ -56,13 +184,19 @@ module GoodData
       attr_reader :title, :datasets, :reports, :metrics, :uploads, :users, :assert_report, :date_dimensions
 
       class << self
-        
+
+        def create_from_data(blueprint)
+          pb = ProjectBuilder.new
+          pb.data = blueprint.to_hash
+          pb
+        end
+
         def create(title, options={}, &block)
           pb = ProjectBuilder.new(title)
           block.call(pb)
           pb
         end
-        
+
       end
 
       def initialize(title)
@@ -129,8 +263,14 @@ module GoodData
         @users << users
       end
 
-      def to_json
-        JSON.pretty_generate(to_hash)
+      def to_json(options={})
+        eliminate_empty = options[:eliminate_empty] || false
+        
+        if eliminate_empty
+          JSON.pretty_generate(to_hash.reject {|k, v| v.is_a?(Enumerable) && v.empty?})
+        else
+          JSON.pretty_generate(to_hash)
+        end
       end
 
       def to_hash
@@ -147,10 +287,14 @@ module GoodData
         }
       end
 
+      def get_dataset(name)
+        datasets.find {|d| d.name == name}
+      end
+
     end
 
     class DashboardBuilder
-      
+
       def initialize(title)
         @title = title
         @tabs = []
@@ -172,7 +316,7 @@ module GoodData
     end
 
     class TabBuilder
-      
+
       def initialize(title)
         @title = title
         @stuff = []
@@ -193,15 +337,35 @@ module GoodData
 
     class SchemaBuilder
 
-      attr_accessor :title, :name
+      attr_accessor :data
+
+      class << self
+        
+        def create_from_data(blueprint)
+          sc = SchemaBuilder.new
+          sc.data = blueprint.to_hash
+          sc
+        end
+
+      end
 
       def initialize(name=nil)
-        @name = name
-        @columns = []
+        @data = {
+          :name => name,
+          :columns => []
+        }
+      end
+
+      def name
+        data[:name]
+      end
+
+      def columns
+        data[:columns]
       end
 
       def add_column(column_def)
-        @columns.push(column_def)
+        columns.push(column_def)
         self
       end
 
@@ -233,21 +397,18 @@ module GoodData
         add_column({ :type => :reference, :name => name}.merge(options))
       end
 
-      def to_schema
-        Schema.new(to_hash)
-      end
-
       def to_json
         JSON.pretty_generate(to_hash)
       end
 
       def to_hash
-        h = {
-          :name => @name,
-          :columns => @columns
-        }
-        h.has_key?(:title) ? h.merge({:title => h[:title]}) : h
+        data
       end
+
+      def to_schema
+        Schema.new(to_hash)
+      end
+
     end
 
     class ProjectCreator
@@ -257,20 +418,20 @@ module GoodData
 
           spec = options[:spec] || fail("You need to provide spec for migration")
           spec = spec.to_hash
-          project = options[:project]
+          
           token = options[:token] || fail("You need to specify token for project creation")
-          new_project = GoodData::Project.create(:title => spec[:title], :auth_token => token)
+          project = options[:project] || GoodData::Project.create(:title => spec[:title], :auth_token => token)
 
           begin
-            GoodData.with_project(new_project) do |p|
-              migrate_date_dimensions(p, spec[:date_dimensions])
-              migrate_datasets(p, spec[:datasets])
+            GoodData.with_project(project) do |p|
+              migrate_date_dimensions(p, spec[:date_dimensions] || [])
+              migrate_datasets(p, spec[:datasets] || [])
               load(p, spec)
-              migrate_metrics(p, spec[:metrics])
-              migrate_reports(p, spec[:reports])
-              migrate_dashboards(p, spec[:dashboards])
-              migrate_users(p, spec[:users])
-              execute_tests(p, spec[:assert_tests])
+              migrate_metrics(p, spec[:metrics] || [])
+              migrate_reports(p, spec[:reports] || [])
+              migrate_dashboards(p, spec[:dashboards] || [])
+              migrate_users(p, spec[:users] || [])
+              execute_tests(p, spec[:assert_tests] || [])
               p
             end
           end
@@ -284,7 +445,9 @@ module GoodData
 
         def migrate_datasets(project, spec)
           spec.each do |ds|
-            project.add_dataset(GoodData::Model::Schema.new(ds))
+            schema = GoodData::Model::Schema.new(ds)
+            project.add_dataset(schema)
+            GoodData::ProjectMetadata["manifest_#{schema.name}"] = schema.to_manifest.to_json
           end
         end
 
@@ -459,55 +622,20 @@ module GoodData
           Tempfile.open('remote_file') do |temp|
             temp << open(path).read
             temp.flush
-            upload_data(temp, project, mode)
+            upload_data(temp, mode)
           end
         else
-          upload_data(path, project, mode)
+          upload_data(path, mode)
         end
       end
 
-      # Load given file into a data set described by the given schema
-      def upload_data(path, project = nil, mode = "FULL")
-        path = path.path if path.respond_to? :path
-
-        inline_data = path.is_a?(String) ? false : true
-
-        project = GoodData.project unless project
-
-        # create a temporary zip file
-        dir = Dir.mktmpdir
-        Zip::File.open("#{dir}/upload.zip", Zip::File::CREATE) do |zip|
-          # TODO make sure schema columns match CSV column names
-          zip.get_output_stream('upload_info.json') { |f| f.puts JSON.pretty_generate(to_manifest(mode)) }
-          if inline_data
-            zip.get_output_stream('data.csv') do |f|
-              path.each do |row|
-                f.puts row.to_csv
-              end
-            end
-          else
-            zip.add('data.csv', path)
-          end
-        end
-
-        # upload it
-        GoodData.upload_to_user_webdav("#{dir}/upload.zip", :directory => File.basename(dir))
-        FileUtils.rm_rf dir
-
-        # kick the load
-        pull = { 'pullIntegration' => File.basename(dir) }
-        link = project.md.links('etl')['pull']
-        task = GoodData.post link, pull
-        while (GoodData.get(task["pullTask"]["uri"])["taskStatus"] === "RUNNING" || GoodData.get(task["pullTask"]["uri"])["taskStatus"] === "PREPARED") do
-          sleep 30
-        end
-        fail "Load Failed" if (GoodData.get(task["pullTask"]["uri"])["taskStatus"] == "ERROR")
-        puts "Done loading"
+      def upload_data(path, mode)
+        GoodData::Model.upload_data(path, to_manifest(mode))
       end
 
       # Generates the SLI manifest describing the data loading
       # 
-      def to_manifest(mode)
+      def to_manifest(mode="FULL")
         {
           'dataSetSLIManifest' => {
             'parts'   => fields.reduce([]) { |memo, f| val = f.to_manifest_part(mode); memo << val unless val.nil?; memo },
@@ -1024,7 +1152,6 @@ module GoodData
         @title = spec[:title] || @name
         @urn = spec[:urn] || "URN:GOODDATA:DATE"
       end
-        
 
       def to_maql_create
         # urn = "urn:chefs_warehouse_fiscal:date"

data/lib/gooddata/models/data_result.rb

@@ -1,36 +1,7 @@
-module GoodData
-
-  class Row < CSV::Row
-    def ==(other)
-       len = length()
-       return false if len != other.length
-       result = true
-
-       len.times do |i|
-         result = false unless convert_field(field(i)) == convert_field(other.field(i))
-       end
-       result
-    end
-
-    private
-    def convert_field(val)
-      if val.is_a?(String) && val.match(/^[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?$/)
-        # Is it a Number?
-        val = val.scan(/[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?/).first
-        val = val.include?('.') ? val.to_f.round : val.to_i
-        return val
-      elsif val.nil? || (val.respond_to?(:strip) && val.strip.empty?)
-        #is ia a String
-        return ''
-      elsif val.respond_to? :round
-        # No idea what that one does
-        return val.round
-      else
-        return val
-      end
-    end
-  end
+class BigDecimal; def pretty_print(p) p.text to_s; end; end
 
+module GoodData
+  
   class DataResult
 
     attr_reader :data
@@ -43,14 +14,17 @@ module GoodData
       puts to_s
     end
 
-    def to_s
+    def to_s(options={})
+      with_indices = options[:index] || false
       a = to_table.to_a
-      a.transpose.unshift((1..a.length).to_a).each_with_index.map{|col, i|
-        col.unshift(i.zero?? nil : i)   # inserts row labels #
+      data = a.transpose
+      data.unshift((1..a.length).to_a) if with_indices
+      data.each_with_index.map{|col, i|
+        col.unshift(i.zero? ? nil : i) if with_indices  # inserts row labels #
         w = col.map{|cell| cell.to_s.length}.max   # w = "column width" #
         col.each_with_index.map{|cell, i|
-          i.zero?? cell.to_s.center(w) : cell.to_s.ljust(w)}   # alligns the column #
-      }.transpose.map{|row| "[#{row.join(' | ')}]"}.join("\n")
+          i.zero? ? cell.to_s.center(w) : cell.to_s.ljust(w)} # alligns the column #
+      }.transpose.map{|row| "[#{row.join(' | ')}]"}.unshift("").join("\n")
     end
 
     def to_table
@@ -72,7 +46,8 @@ module GoodData
     end
 
     def assemble_table
-      @table = CSV::Table.new([GoodData::Row.new([],[],false)])
+      @table = [[]]
+      # CSV::Table.new([GoodData::Row.new([],[],false)])
     end
 
     def to_table
@@ -92,69 +67,71 @@ module GoodData
     end
   end
 
-  class SFDataResult < DataResult
-
-    def initialize(data, options = {})
-      super(data)
-      @options = options
-      assemble_table
-    end
-
-    def assemble_table
-      sf_data = data[:queryResponse][:result][:records]
-      sf_data = sf_data.is_a?(Hash) ? [sf_data] : sf_data
-      if @options[:soql]
-        # puts @options[:soql]
-        fields = @options[:soql].strip.match(/SELECT (.*) FROM/i)[1]
-        @headers = fields.strip.split(",").map do |item|
-          item.strip.split(/\s/)
-        end.map do |item|
-          item.last.to_sym
-        end
-        # pp @headers
-      elsif @options[:headers]
-        @headers = @options[:headers]
-      else
-        @headers = sf_data.first.keys - [:type, :Id]
-      end
-      @table = CSV::Table.new(sf_data.collect do |line|
-        GoodData::Row.new([], @headers.map {|h| line[h] || ' '}, false)
-      end)
-    rescue
-      fail "Unable to assemble the table. Either the data provided are empty or the SOQL is malformed."
-    end
-
-    def to_table
-      @table
-    end
-
-    def == (otherDataResult)
-      result = true
-      len =  @table.length
-      other_table = otherDataResult.to_table
-      if len != other_table.length
-        # puts "TABLES ARE OF DIFFERENT SIZES"
-        return false
-      end
-
-      diff(otherDataResult).empty?() ? true : false
-
-    end
-
-    def diff(otherDataResult)
-      other_table = otherDataResult.to_table
-      differences = []
-
-      @table.each do |row|
-        differences << row unless other_table.detect {|r| r == row}
-      end
-      differences
-    end
-
-  end
+  # class SFDataResult < DataResult
+  # 
+  #     def initialize(data, options = {})
+  #       super(data)
+  #       @options = options
+  #       assemble_table
+  #     end
+  # 
+  #     def assemble_table
+  #       sf_data = data[:queryResponse][:result][:records]
+  #       sf_data = sf_data.is_a?(Hash) ? [sf_data] : sf_data
+  #       if @options[:soql]
+  #         # puts @options[:soql]
+  #         fields = @options[:soql].strip.match(/SELECT (.*) FROM/i)[1]
+  #         @headers = fields.strip.split(",").map do |item|
+  #           item.strip.split(/\s/)
+  #         end.map do |item|
+  #           item.last.to_sym
+  #         end
+  #         # pp @headers
+  #       elsif @options[:headers]
+  #         @headers = @options[:headers]
+  #       else
+  #         @headers = sf_data.first.keys - [:type, :Id]
+  #       end
+  #       @table = CSV::Table.new(sf_data.collect do |line|
+  #         GoodData::Row.new([], @headers.map {|h| line[h] || ' '}, false)
+  #       end)
+  #     rescue
+  #       fail "Unable to assemble the table. Either the data provided are empty or the SOQL is malformed."
+  #     end
+  # 
+  #     def to_table
+  #       @table
+  #     end
+  # 
+  #     def == (otherDataResult)
+  #       result = true
+  #       len =  @table.length
+  #       other_table = otherDataResult.to_table
+  #       if len != other_table.length
+  #         # puts "TABLES ARE OF DIFFERENT SIZES"
+  #         return false
+  #       end
+  # 
+  #       diff(otherDataResult).empty?() ? true : false
+  # 
+  #     end
+  # 
+  #     def diff(otherDataResult)
+  #       other_table = otherDataResult.to_table
+  #       differences = []
+  # 
+  #       @table.each do |row|
+  #         differences << row unless other_table.detect {|r| r == row}
+  #       end
+  #       differences
+  #     end
+  # 
+  #   end
 
   class ReportDataResult < DataResult
 
+    ROW_LIMIT = 10000000
+
     attr_reader :row_headers, :column_headers, :table, :headers_height, :headers_width
 
     def initialize(data)
@@ -170,16 +147,32 @@ module GoodData
     end
 
     def without_column_headers
-      @table = table.transpose[headers_height, 1000000].transpose
+      @table = table.transpose[headers_height, ROW_LIMIT].transpose
       self
     end
 
+    def to_data_table
+      table.transpose[headers_height, ROW_LIMIT].transpose[headers_width, ROW_LIMIT]
+    end
+
     def each_line
       table.transpose.each {|line| yield line}
     end
 
+    def to_a
+      table.to_a
+    end
+
     def to_table
-      CSV::Table.new(table.transpose.map {|line| GoodData::Row.new([], line.map {|item| item || ' '}, false)})
+      table.transpose
+    end
+
+    def [](index)
+      table[index]
+    end
+
+    def include_row?(row)
+      to_table.include?(row)
     end
 
     def == (otherDataResult)
@@ -275,7 +268,7 @@ module GoodData
         (row_headers.size).times do |j|
           table[headers_width + i] ||= []
   #        puts "[#{headers_width + i}, #{headers_height + j}] [#{i}][#{j}]=#{xtab_data[j][i]}"
-          table[headers_width + i][headers_height + j] = xtab_data[j][i]
+          table[headers_width + i][headers_height + j] = BigDecimal(xtab_data[j][i])
         end
       end
     end