gooddata 0.6.0.pre9 → 0.6.0.pre10

data/lib/gooddata/bricks/middleware/gooddata_middleware.rb

@@ -7,7 +7,7 @@ module GoodData::Bricks
       logger = params[:gdc_logger]
       token_name = :GDC_SST
       protocol_name = :GDC_PROTOCOL
-      server_name = :GDC_SERVER
+      server_name = :GDC_HOSTNAME
       project_id = params[:GDC_PROJECT_ID]
 
       fail "SST (SuperSecureToken) not present in params" if params[token_name].nil?

data/lib/gooddata/client.rb

@@ -10,39 +10,47 @@ else
   FasterCSV = CSV
 end
 
-# Metadata packages, such as report.rb, require this to be loaded first
-require File.dirname(__FILE__) + '/models/metadata.rb'
+# Initializes required dynamically loaded classes
+def init_gd_module()
+  # Metadata packages, such as report.rb, require this to be loaded first
+  require File.dirname(__FILE__) + '/models/metadata.rb'
 
-Dir[File.dirname(__FILE__) + '/models/*.rb'].each { |file| require file }
-Dir[File.dirname(__FILE__) + '/collections/*.rb'].each { |file| require file }
+  # Load models from models folder
+  Dir[File.dirname(__FILE__) + '/models/*.rb'].each { |file| require file }
 
-# = GoodData API wrapper
+  # Load collections
+  Dir[File.dirname(__FILE__) + '/collections/*.rb'].each { |file| require file }
+end
+
+init_gd_module()
+
+# # GoodData API wrapper
 #
 # A convenient Ruby wrapper around the GoodData RESTful API.
 #
 # The best documentation for the API can be found using these resources:
+#
 # * http://developer.gooddata.com/api
 # * https://secure.gooddata.com/gdc
 #
-# == Usage
+# ## Usage
 #
 # To communicate with the API you first need a personal GoodData account.
-# {Sign up here}[https://secure.gooddata.com/registration.html] if you havent already.
+# [Sign up here](https://secure.gooddata.com/registration.html) if you havent already.
 #
-# Now it is just a matter of initializing the GoodData connection via the connect
-# method:
+# Now it is just a matter of initializing the GoodData connection via the connect method:
 #
-#   GoodData.connect 'gooddata_user', 'gooddata_password'
+#     GoodData.connect 'gooddata_user', 'gooddata_password'
 #
 # This GoodData object can now be utalized to retrieve your GoodData profile, the available
 # projects etc.
 #
-# == Logging
+# ## Logging
 #
-#   GoodData.logger = Logger.new(STDOUT)
+#     GoodData.logger = Logger.new(STDOUT)
 #
 # For details about the logger options and methods, see the
-# {Logger module documentation}[http://www.ruby-doc.org/stdlib/libdoc/logger/rdoc].
+# (Logger module documentation)[http://www.ruby-doc.org/stdlib/libdoc/logger/rdoc].
 #
 module GoodData
   module Threaded
@@ -52,6 +60,7 @@ module GoodData
     end
   end
 
+  # Dummy implementation of logger
   class NilLogger
     def debug(*args) ; end
     alias :info :debug
@@ -59,6 +68,7 @@ module GoodData
     alias :error :debug
   end
 
+  # Assigns global/default GoodData project
   def project=(project)
     GoodData.project = project
     GoodData.project
@@ -70,21 +80,24 @@ module GoodData
 
     RELEASE_INFO_PATH = '/gdc/releaseInfo'
 
+    # Version
     def version
       VERSION
     end
 
-    def gem_version_string
+    # Identifier of gem version
+    # @return Formatted gem version
+    def gem_version_string()
       "gooddata-gem/#{version}"
     end
 
     # Connect to the GoodData API
     #
-    # === Parameters
-    #
-    # * +user+ - A GoodData username
-    # * +password+ - A GoodData password
+    # @param options
+    # @param second_options
+    # @param third_options
     #
+    #    Goodd
     def connect(options=nil, second_options=nil, third_options={})
       if options.is_a? Hash
         fail "You have to provide login and password" if ((options[:login].nil? || options[:login].empty?) && (options[:password].nil? || options[:password].empty?))
@@ -94,36 +107,34 @@ module GoodData
         fail "You have to provide login and password" if ((options.nil? || options.empty?) && (second_options.nil? || second_options.empty?))
         threaded[:connection] = Connection.new(options, second_options, third_options)
       end
-      
     end
 
+    # Turn logging on
     def logging_on
       if logger.is_a? NilLogger
         GoodData::logger = Logger.new(STDOUT)
       end
     end
 
+    # Turn logging off
     def logging_off
       GoodData::logger = NilLogger.new
     end
 
 
     # Hepler for starting with SST easier
-    # === Parameters
     #
-    # * +token+ - SST token
-    # * +options+ - options get routed to connect eventually so everything that you can use there should be possible to use here.
+    # @param token SST token
+    # @param options Options get routed to connect eventually so everything that you can use there should be possible to use here.
     #
     def connect_with_sst(token, options={})
       create_authenticated_connection(options.merge({:cookies => {"GDCAuthSST" => token}}))
     end
 
     # This method is aimed at creating an authenticated connection in case you do not hae pass/login but you have SST
-    # === Parameters
     #
-    # * +options+ - :server => optional GD server uri. If nil it secure will be used 
-    # * +options+ - :cookies => you can specify a hash of cookies
-    # 
+    # @param options :server => optional GD server uri, If nil it secure will be used. :cookies => you can specify a hash of cookies
+    #
     def create_authenticated_connection(options={})
       connect(options)
       server_cookies = options[:cookies]
@@ -132,6 +143,10 @@ module GoodData
       connection
     end
 
+    # Perform block in context of another project than currently set
+    #
+    # @param project Project to use
+    # @param bl Block to be performed
     def with_project(project, &bl)
       fail "You have to specify a project when using with_project" if project.nil? || (project.is_a?(String) && project.empty?)
       old_project = GoodData.project
@@ -145,24 +160,21 @@ module GoodData
       end
     end
 
-    # Returns the active GoodData connection earlier initialized via
-    # GoodData.connect call
+    # Returns the active GoodData connection earlier initialized via GoodData.connect call
     #
     # @see GoodData.connect
-    #
     def connection
       threaded[:connection] || raise("Please authenticate with GoodData.connect first")
     end
 
     # Sets the active project
     #
-    # === Parameters
+    # @param project A project identifier
     #
-    # * +project+ - a project identifier
+    # ### Examples
     #
-    # === Examples
+    # The following calls are equivalent
     #
-    # The following calls are equivalent:
     # * GoodData.project = 'afawtv356b6usdfsdf34vt'
     # * GoodData.use 'afawtv356b6usdfsdf34vt'
     # * GoodData.use '/gdc/projects/afawtv356b6usdfsdf34vt'
@@ -190,12 +202,12 @@ module GoodData
     #
     # Retuns the JSON response formatted as a Hash object.
     #
-    # === Parameters
+    # @param path The HTTP path on the GoodData server (must be prefixed with a forward slash)
+    #
+    # ### Examples
     #
-    # * +path+ - The HTTP path on the GoodData server (must be prefixed with a forward slash)
-    # === Examples
+    #     GoodData.get '/gdc/projects'
     #
-    #   GoodData.get '/gdc/projects'
     def get(path, options = {})
       connection.get(path, options)
     end
@@ -204,14 +216,13 @@ module GoodData
     #
     # Retuns the JSON response formatted as a Hash object.
     #
-    # === Parameters
-    #
-    # * +path+ - The HTTP path on the GoodData server (must be prefixed with a forward slash)
-    # * +data+ - The payload data in the format of a Hash object
+    # @param path The HTTP path on the GoodData server (must be prefixed with a forward slash)
+    # @param data The payload data in the format of a Hash object
     #
-    # === Examples
+    # ### Examples
     #
     #   GoodData.post '/gdc/projects', { ... }
+    #
     def post(path, data, options = {})
       connection.post path, data, options
     end
@@ -220,14 +231,15 @@ module GoodData
     #
     # Retuns the JSON response formatted as a Hash object.
     #
-    # === Parameters
+    # ### Parameters
     #
-    # * +path+ - The HTTP path on the GoodData server (must be prefixed with a forward slash)
-    # * +data+ - The payload data in the format of a Hash object
+    # @param path The HTTP path on the GoodData server (must be prefixed with a forward slash)
+    # @param data The payload data in the format of a Hash object
     #
-    # === Examples
+    # ### Examples
     #
     #   GoodData.put '/gdc/projects', { ... }
+    #
     def put(path, data, options = {})
       connection.put path, data, options
     end
@@ -236,13 +248,12 @@ module GoodData
     #
     # Retuns the JSON response formatted as a Hash object.
     #
-    # === Parameters
+    # @param path The HTTP path on the GoodData server (must be prefixed with a forward slash)
     #
-    # * +path+ - The HTTP path on the GoodData server (must be prefixed with a forward slash)
+    # ### Examples
     #
-    # === Examples
+    #     GoodData.delete '/gdc/project/1'
     #
-    #   GoodData.delete '/gdc/project/1'
     def delete(path, options = {})
       connection.delete path, options
     end
@@ -306,10 +317,11 @@ module GoodData
     # For some serious logging, set the logger instance using
     # the logger= method
     #
-    # === Example
+    # ### Example
+    #
+    #     require 'logger'
+    #     GoodData.logger = Logger.new(STDOUT)
     #
-    #   require 'logger'
-    #   GoodData.logger = Logger.new(STDOUT)
     def logger
       @logger ||= NilLogger.new
     end

data/lib/gooddata/commands/commands.rb

@@ -0,0 +1,9 @@
+require './api'
+require './auth'
+require './base'
+require './datasets'
+require './process'
+require './profile'
+require './projects'
+require './runners'
+require './scaffold'

data/lib/gooddata/commands/process.rb

@@ -19,23 +19,25 @@ module GoodData::Command
     def self.deploy(dir, options={})
       verbose = options[:verbose] || false
       GoodData.with_project(options[:project_id]) do
-        deploy_graph(dir, options.merge({:files_to_exclude => [options[:params]]}))
+        params = options[:params].nil? ? [] : [options[:params]]
+        deploy_graph(dir, options.merge({:files_to_exclude => params}))
       end
     end
 
     def self.with_deploy(dir, options={}, &block) 
       verbose = options[:verbose] || false
       GoodData.with_project(options[:project_id]) do
+      params = options[:params].nil? ? [] : [options[:params]]
         if block
           begin
-            res = deploy_graph(dir, options.merge({:files_to_exclude => [options[:params]]}))
+            res = deploy_graph(dir, options.merge({:files_to_exclude => params}))
             block.call(res)
           ensure
-            # self_link = res["process"]["links"]["self"]
-            # GoodData.delete(self_link)
+            self_link = res && res["process"]["links"]["self"]
+            GoodData.delete(self_link)
           end
         else
-          deploy_graph(dir, options.merge({:files_to_exclude => [options[:params]]}))
+          deploy_graph(dir, options.merge({:files_to_exclude => params}))
         end
       end
     end
@@ -100,8 +102,7 @@ module GoodData::Command
       fail "\"#{dir}\" is not a directory" unless dir.directory?
       files_to_exclude = options[:files_to_exclude].map {|p| Pathname(p)}
 
-      # project_id = options[:project_id] || fail("Project Id has to be specified")
-      
+      project_id = options[:project_id] || fail("Project Id has to be specified")
 
       type = options[:type] || "GRAPH"
       deploy_name = options[:name]
@@ -141,7 +142,7 @@ module GoodData::Command
           GoodData.put("/gdc/projects/#{GoodData.project.pid}/dataload/processes/#{process_id}", data)
         end
       end
-      puts HighLine::color("Deploy DONE #{dir}", HighLine::BOLD) if verbose
+      puts HighLine::color("Deploy DONE #{dir}", HighLine::GREEN) if verbose
       res
     end
 

data/lib/gooddata/commands/projects.rb

@@ -66,6 +66,35 @@ module GoodData::Command
       p.delete
     end
 
+    def self.get_spec_and_project_id(base_path)
+      goodfile_path = GoodData::Helpers.find_goodfile(Pathname(base_path))
+      fail "Goodfile could not be located in any parent directory. Please make sure you are inside a gooddata project folder." if goodfile_path.nil?
+      goodfile = JSON.parse(File.read(goodfile_path), :symbolize_names => true)
+      spec_path = goodfile[:model] || fail("You need to specify the path of the build spec")
+      fail "Model path provided in Goodfile \"#{spec_path}\" does not exist" unless File.exist?(spec_path) && !File.directory?(spec_path)
+
+      spec_path = Pathname(spec_path)
+
+      content = File.read(spec_path)
+      spec = if (spec_path.extname == ".rb")
+               eval(content)
+             elsif (spec_path.extname == ".json")
+               JSON.parse(spec_path, :symbolize_names => true)
+             end
+      [spec, goodfile[:project_id]]
+    end
+
+    def self.update(options={})
+      project = options[:project]
+      project_id = project && project.pid
+      fail "You have to provide 'project_id'. You can either provide it through -p flag or even better way is to fill it in in your Goodfile under key \"project_id\". If you just started a project you have to create it first. One way might be through \"gooddata project build\"" if project_id.nil? || project_id.empty?
+      GoodData::Model::ProjectCreator.migrate(:spec => options[:spec], :project => project_id)
+    end
+
+    def self.build(options={})
+      GoodData::Model::ProjectCreator.migrate(:spec => options[:spec], :token => options[:token])
+    end
+
   end
 end
 

data/lib/gooddata/commands/runners.rb

@@ -29,7 +29,7 @@ script_body = <<-script_body
           :GDC_SST => \"#{sst}\",
           :GDC_PROJECT_ID => \"#{pid}\",
           :GDC_PROTOCOL => \"#{scheme}\",
-          :GDC_SERVER => \"#{hostname}\",
+          :GDC_HOSTNAME => \"#{hostname}\",
           :GDC_LOGGER_FILE => STDOUT
         }.merge(#{params})
         eval(File.read(\"./main.rb\"))

data/lib/gooddata/connection.rb

@@ -1,7 +1,5 @@
 require 'json'
-
-# silence the parenthesis warning in rest-client 1.6.1
-old_verbose, $VERBOSE = $VERBOSE, nil ; require 'rest-client' ; $VERBOSE = old_verbose
+require 'rest-client'
 
 module GoodData
 
@@ -368,7 +366,11 @@ module GoodData
       new_params = Marshal.load(Marshal.dump(params))
       GoodData::Helpers.hash_dfs(new_params) do |k, key|
         keys.each do |key_to_scrub|
-          k[key_to_scrub] = ("*" * k[key_to_scrub].length) if k && k.has_key?(key_to_scrub)
+          begin
+            k[key_to_scrub] = ("*" * k[key_to_scrub].length) if k && k.has_key?(key_to_scrub) && k[key_to_scrub]
+          rescue
+            binding.pry
+          end
         end
       end      
       new_params

data/lib/gooddata/exceptions.rb

@@ -1,5 +1,6 @@
 module GoodData
-  
+
+  # Project Not Found
   class ProjectNotFound < RestClient::ResourceNotFound
     
   end

data/lib/gooddata/helpers.rb

@@ -26,7 +26,7 @@ module GoodData::Helpers
        end
        pwd = pwd.parent
     end until root == pwd
-    fail "Goodfile not found in #{pwd.to_s} or any parent up to #{root.to_s}"
+    nil
   end
 
   def self.hash_dfs(thing, &block)

data/lib/gooddata/model.rb

@@ -34,8 +34,8 @@ module GoodData
 
     class << self
       def add_dataset(name, columns, project = nil)
-          Schema.new('columns' => columns, 'name' => name)
-        add_schema(schema , project)
+        Schema.new('columns' => columns, 'name' => name)
+        add_schema(schema, project)
       end
 
       def add_schema(schema, project = nil)
@@ -46,7 +46,7 @@ module GoodData
         project = GoodData.project unless project
         ldm_links = GoodData.get project.md[LDM_CTG]
         ldm_uri = Links.new(ldm_links)[LDM_MANAGE_CTG]
-        GoodData.post ldm_uri, { 'manage' => { 'maql' => schema.to_maql_create } }
+        GoodData.post ldm_uri, {'manage' => {'maql' => schema.to_maql_create}}
       end
 
       # Load given file into a data set described by the given schema
@@ -80,7 +80,7 @@ module GoodData
         end
 
         # kick the load
-        pull = { 'pullIntegration' => File.basename(dir) }
+        pull = {'pullIntegration' => File.basename(dir)}
         link = project.md.links('etl')['pull']
         task = GoodData.post link, pull
         while (GoodData.get(task["pullTask"]["uri"])["taskStatus"] === "RUNNING" || GoodData.get(task["pullTask"]["uri"])["taskStatus"] === "PREPARED") do
@@ -92,16 +92,17 @@ module GoodData
           js = JSON.parse(s.string)
           fail "Load Failed with error #{JSON.pretty_generate(js)}"
         end
-        puts "Done loading"
       end
 
       def merge_dataset_columns(a_schema_blueprint, b_schema_blueprint)
-         d = Marshal.load(Marshal.dump(a_schema_blueprint))
-         d[:columns] = d[:columns] + b_schema_blueprint[:columns]
-         d[:columns].uniq!
-         columns_that_failed_to_merge = d[:columns].group_by {|x| x[:name]}.map {|k, v| [k, v.count]}.find_all {|x| x[1] > 1}
-         fail "Columns #{columns_that_failed_to_merge} failed to merge. When merging columns with the same name they have to be identical." unless columns_that_failed_to_merge.empty?
-         d
+        a_schema_blueprint = a_schema_blueprint.to_hash
+        b_schema_blueprint = b_schema_blueprint.to_hash
+        d = Marshal.load(Marshal.dump(a_schema_blueprint))
+        d[:columns] = d[:columns] + b_schema_blueprint[:columns]
+        d[:columns].uniq!
+        columns_that_failed_to_merge = d[:columns].group_by {|x| x[:name]}.map {|k, v| [k, v.count]}.find_all {|x| x[1] > 1}
+        fail "Columns #{columns_that_failed_to_merge} failed to merge. When merging columns with the same name they have to be identical." unless columns_that_failed_to_merge.empty?
+        d
       end
 
     end
@@ -110,6 +111,14 @@ module GoodData
 
       attr_accessor :data
 
+      def self.from_json(spec)
+        if spec.is_a?(String)
+          ProjectBlueprint.new(JSON.parse(File.read(spec), :symbolize_names => true))
+        else
+          ProjectBlueprint.new(spec)
+        end
+      end
+
       def change(&block)
         builder = ProjectBuilder.create_from_data(self)
         block.call(builder)
@@ -119,11 +128,29 @@ module GoodData
       end
 
       def datasets
-        data[:datasets].map {|d| SchemaBlueprint.new(d)}
+        data[:datasets].map { |d| SchemaBlueprint.new(d) }
+      end
+
+      def add_dataset(a_dataset, index=nil)
+        if index.nil? || index > datasets.length
+          data[:datasets] << a_dataset.to_hash
+        else
+          data[:datasets].insert(index, a_dataset.to_hash)
+        end
+      end
+
+      def remove_dataset(dataset_name)
+        x = data[:datasets].find {|d| d[:name] == dataset_name}
+        index = data[:datasets].index(x)
+        data[:datasets].delete_at(index)
+      end
+
+      def date_dimensions
+        data[:date_dimensions]
       end
 
       def get_dataset(name)
-        ds = data[:datasets].find {|d| d[:name] == name}
+        ds = data[:datasets].find { |d| d[:name] == name }
         SchemaBlueprint.new(ds) unless ds.nil?
       end
 
@@ -135,7 +162,7 @@ module GoodData
         if datasets.count == 1
           []
         else
-          x = datasets.reduce([]) {|memo, schema| schema.has_anchor? ? memo << [schema.name, schema.anchor[:name]] : memo }          
+          x = datasets.reduce([]) { |memo, schema| schema.has_anchor? ? memo << [schema.name, schema.anchor[:name]] : memo }
           refs = datasets.reduce([]) do |memo, dataset|
             memo.concat(dataset.references)
           end
@@ -147,12 +174,55 @@ module GoodData
 
       def model_valid?
         errors = model_validate
-        errors.empty? ? true :false
+        errors.empty? ? true : false
+      end
+
+      def merge!(a_blueprint)
+        temp_blueprint = dup
+        a_blueprint.datasets.each do |dataset|
+          local_dataset = temp_blueprint.get_dataset(dataset.name)
+          if local_dataset.nil?
+            temp_blueprint.add_dataset(dataset.dup)
+          else
+            index = temp_blueprint.datasets.index(local_dataset)
+            local_dataset.merge!(dataset)
+            temp_blueprint.remove_dataset(local_dataset.name)
+            temp_blueprint.add_dataset(local_dataset, index)
+          end
+        end
+        @data = temp_blueprint.data
+        self
+      end
+
+      def dup
+        deep_copy = Marshal.load(Marshal.dump(data))
+        ProjectBlueprint.new(deep_copy)
       end
 
       def title
         data[:title]
       end
+
+      def to_wire_model
+        {
+          "diffRequest" => {
+              "targetModel" => {
+                  "projectModel" => {
+                      "datasets" => datasets.map {|d| d.to_wire_model},
+                      "dateDimensions" => date_dimensions.map {|d|
+                        {
+                            "dateDimension" => {
+                                "name" => d[:name],
+                                "title" => d[:title] || d[:name].humanize
+                            }
+                        }}
+                      }}}}
+      end
+
+      def to_hash
+        @data
+      end
+
     end
 
     class SchemaBlueprint
@@ -173,10 +243,17 @@ module GoodData
 
       def upload(source, options={})
         project = options[:project] || GoodData.project
+        fail "You have to specify a project into which you want to load." if project.nil?
         mode = options[:load] || "FULL"
         project.upload(source, to_schema, mode)
       end
 
+      def merge!(a_blueprint)
+        new_blueprint = GoodData::Model.merge_dataset_columns(self, a_blueprint)
+        @data = new_blueprint
+        self
+      end
+
       def name
         data[:name]
       end
@@ -194,7 +271,7 @@ module GoodData
       end
 
       def has_anchor?
-        columns.any? {|c| c[:type] == :anchor}
+        columns.any? { |c| c[:type].to_s == "anchor" }
       end
 
       def anchor
@@ -205,19 +282,29 @@ module GoodData
         find_column_by_type(:reference)
       end
 
+      def attributes
+        find_column_by_type(:attribute)
+      end
+
+      def facts
+        find_column_by_type(:fact)
+      end
+
       def find_column_by_type(type, all=:all)
+        type = type.to_s
         if all == :all
-          columns.find_all {|c| c[:type] == type}
+          columns.find_all { |c| c[:type].to_s == type }
         else
-          columns.find {|c| c[:type] == type}
+          columns.find { |c| c[:type].to_s == type }
         end
       end
 
       def find_column_by_name(type, all=:all)
+        type = type.to_s
         if all == :all
-          columns.find_all {|c| c[:name] == type}
+          columns.find_all { |c| c[:name].to_s == type }
         else
-          columns.find {|c| c[:name] == type}
+          columns.find { |c| c[:name].to_s == type }
         end
       end
 
@@ -228,12 +315,12 @@ module GoodData
       def to_manifest
         to_schema.to_manifest
       end
-      
+
       def pretty_print(printer)
-        printer.text "Schema <#{object_id}>:\n" 
+        printer.text "Schema <#{object_id}>:\n"
         printer.text " Name: #{name}\n"
         printer.text " Columns: \n"
-        printer.text columns.map {|c| "  #{c[:name]}: #{c[:type]}"}.join("\n")
+        printer.text columns.map { |c| "  #{c[:name]}: #{c[:type]}" }.join("\n")
       end
 
       def dup
@@ -241,6 +328,14 @@ module GoodData
         SchemaBlueprint.new(deep_copy)
       end
 
+      def to_wire_model
+        to_schema.to_wire_model
+      end
+
+      def ==(other)
+        to_hash == other.to_hash
+      end
+
     end
 
     class ProjectBuilder
@@ -282,8 +377,8 @@ module GoodData
       def add_dataset(name, &block)
         builder = GoodData::Model::SchemaBuilder.new(name)
         block.call(builder)
-        if @datasets.any? {|item| item[:name] == name}
-          ds = @datasets.find {|item| item[:name] == name}
+        if @datasets.any? { |item| item[:name] == name }
+          ds = @datasets.find { |item| item[:name] == name }
           index = @datasets.index(ds)
           stuff = GoodData::Model.merge_dataset_columns(ds, builder.to_hash)
           @datasets.delete_at(index)
@@ -325,9 +420,9 @@ module GoodData
         mode = options[:mode] || "FULL"
         dataset = options[:dataset]
         @uploads << {
-          :source => data,
-          :mode => mode,
-          :dataset => dataset
+            :source => data,
+            :mode => mode,
+            :dataset => dataset
         }
       end
 
@@ -337,9 +432,9 @@ module GoodData
 
       def to_json(options={})
         eliminate_empty = options[:eliminate_empty] || false
-        
+
         if eliminate_empty
-          JSON.pretty_generate(to_hash.reject {|k, v| v.is_a?(Enumerable) && v.empty?})
+          JSON.pretty_generate(to_hash.reject { |k, v| v.is_a?(Enumerable) && v.empty? })
         else
           JSON.pretty_generate(to_hash)
         end
@@ -347,20 +442,20 @@ module GoodData
 
       def to_hash
         {
-          :title => @title,
-          :datasets => @datasets,
-          :uploads => @uploads,
-          :dashboards => @dashboards,
-          :metrics => @metrics,
-          :reports => @reports,
-          :users => @users,
-          :assert_tests => @assert_tests,
-          :date_dimensions => @date_dimensions
+            :title => @title,
+            :datasets => @datasets,
+            :uploads => @uploads,
+            :dashboards => @dashboards,
+            :metrics => @metrics,
+            :reports => @reports,
+            :users => @users,
+            :assert_tests => @assert_tests,
+            :date_dimensions => @date_dimensions
         }
       end
 
       def get_dataset(name)
-        datasets.find {|d| d.name == name}
+        datasets.find { |d| d.name == name }
       end
 
     end
@@ -381,8 +476,8 @@ module GoodData
 
       def to_hash
         {
-          :name => @name,
-          :tabs => @tabs.map{|tab| tab.to_hash}
+            :name => @name,
+            :tabs => @tabs.map { |tab| tab.to_hash }
         }
       end
     end
@@ -400,8 +495,8 @@ module GoodData
 
       def to_hash
         {
-          :title => @title,
-          :items => @stuff
+            :title => @title,
+            :items => @stuff
         }
       end
 
@@ -412,7 +507,7 @@ module GoodData
       attr_accessor :data
 
       class << self
-        
+
         def create_from_data(blueprint)
           sc = SchemaBuilder.new
           sc.data = blueprint.to_hash
@@ -423,8 +518,8 @@ module GoodData
 
       def initialize(name=nil)
         @data = {
-          :name => name,
-          :columns => []
+            :name => name,
+            :columns => []
         }
       end
 
@@ -442,31 +537,31 @@ module GoodData
       end
 
       def add_anchor(name, options={})
-        add_column({ :type => :anchor, :name => name}.merge(options))
+        add_column({:type => :anchor, :name => name}.merge(options))
         self
       end
 
       def add_attribute(name, options={})
-        add_column({ :type => :attribute, :name => name}.merge(options))
+        add_column({:type => :attribute, :name => name}.merge(options))
         self
       end
 
       def add_fact(name, options={})
-        add_column({ :type => :fact, :name => name}.merge(options))
+        add_column({:type => :fact, :name => name}.merge(options))
         self
       end
 
       def add_label(name, options={})
-        add_column({ :type => :label, :name => name}.merge(options))
+        add_column({:type => :label, :name => name}.merge(options))
         self
       end
 
       def add_date(name, options={})
-        add_column({ :type => :date, :name => name}.merge(options))
+        add_column({:type => :date, :name => name}.merge(options))
       end
 
       def add_reference(name, options={})
-        add_column({ :type => :reference, :name => name}.merge(options))
+        add_column({:type => :reference, :name => name}.merge(options))
       end
 
       def to_json
@@ -484,20 +579,21 @@ module GoodData
     end
 
     class ProjectCreator
-      
+
       class << self
         def migrate(options={})
 
           spec = options[:spec] || fail("You need to provide spec for migration")
           spec = spec.to_hash
-          
-          token = options[:token] || fail("You need to specify token for project creation")
+
+          token = options[:token]
           project = options[:project] || GoodData::Project.create(:title => spec[:title], :auth_token => token)
+          fail("You need to specify token for project creation") if token.nil? && project.nil?
 
           begin
             GoodData.with_project(project) do |p|
-              migrate_date_dimensions(p, spec[:date_dimensions] || [])
-              migrate_datasets(p, spec[:datasets] || [])
+              # migrate_date_dimensions(p, spec[:date_dimensions] || [])
+              migrate_datasets(p, spec)
               load(p, spec)
               migrate_metrics(p, spec[:metrics] || [])
               migrate_reports(p, spec[:reports] || [])
@@ -516,9 +612,31 @@ module GoodData
         end
 
         def migrate_datasets(project, spec)
-          spec.each do |ds|
-            schema = GoodData::Model::Schema.new(ds)
-            project.add_dataset(schema)
+          bp = ProjectBlueprint.new(spec)
+          # schema = Schema.load(schema) unless schema.respond_to?(:to_maql_create)
+          # project = GoodData.project unless project
+          result = GoodData.post("/gdc/projects/#{GoodData.project.pid}/model/diff", bp.to_wire_model)
+          link = result["asyncTask"]["link"]["poll"]
+          response = GoodData.get(link, :process => false)
+          # pp response
+          while response.code != 200
+            sleep 1
+            GoodData.connection.retryable(:tries => 3, :on => RestClient::InternalServerError) do
+              sleep 1
+              response = GoodData.get(link, :process => false)
+              # pp response
+            end
+          end
+          response = GoodData.get(link)
+          ldm_links = GoodData.get project.md[LDM_CTG]
+          ldm_uri = Links.new(ldm_links)[LDM_MANAGE_CTG]
+          chunks = response["projectModelDiff"]["updateScripts"].find_all {|script| script["updateScript"]["preserveData"] == true && script["updateScript"]["cascadeDrops"] == false}.map {|x| x["updateScript"]["maqlDdlChunks"]}.flatten
+          chunks.each do |chunk|
+            GoodData.post ldm_uri, { 'manage' => { 'maql' => chunk } }
+          end
+
+          bp.datasets.each do |ds|
+            schema = ds.to_schema
             GoodData::ProjectMetadata["manifest_#{schema.name}"] = schema.to_manifest.to_json
           end
         end
@@ -549,9 +667,11 @@ module GoodData
         end
 
         def load(project, spec)
-          spec[:uploads].each do |load|
-            schema = GoodData::Model::Schema.new(spec[:datasets].detect {|d| d[:name] == load[:dataset]})
-            project.upload(load[:source], schema, load[:mode])
+          if spec.has_key?(:uploads)
+            spec[:uploads].each do |load|
+              schema = GoodData::Model::Schema.new(spec[:datasets].detect {|d| d[:name] == load[:dataset]})
+              project.upload(load[:source], schema, load[:mode])
+            end
           end
         end
 
@@ -563,7 +683,7 @@ module GoodData
         end
       end
     end
-    
+
     class MdObject
       attr_accessor :name, :title
 
@@ -590,7 +710,7 @@ module GoodData
     # model abstractions.
     #
     class Schema < MdObject
-      attr_reader :fields, :attributes, :facts, :folders, :references, :labels, :name, :title
+      attr_reader :fields, :attributes, :facts, :folders, :references, :labels, :name, :title, :anchor
 
       def self.load(file)
         Schema.new JSON.load(open(file))
@@ -599,13 +719,13 @@ module GoodData
       def initialize(config, name = nil)
         super()
         @fields = []
-        @attributes = {}
-        @facts = {}
+        @attributes = []
+        @facts = []
         @folders = {
-          :facts      => {},
-          :attributes => {}
+            :facts => {},
+            :attributes => {}
         }
-        @references = {}
+        @references = []
         @labels = []
 
         config[:name] = name unless config[:name]
@@ -626,7 +746,7 @@ module GoodData
           when "date"
             add_date c
           when "anchor"
-            set_connection_point c
+            set_anchor c
           when "label"
             add_label c
           when "reference"
@@ -635,7 +755,7 @@ module GoodData
             fail "Unexpected type #{c[:type]} in #{c.inspect}"
           end
         end
-        @connection_point = RecordsOf.new(nil, self) unless @connection_point
+        @anchor = Anchor.new(nil, self) unless @anchor
       end
 
       def type_prefix
@@ -654,7 +774,7 @@ module GoodData
       #
       def to_maql_drop
         maql = ""
-        [ attributes, facts ].each do |obj|
+        [attributes, facts].each do |obj|
           maql += obj.to_maql_drop
         end
         maql += "DROP {#{self.identifier}};\n"
@@ -666,7 +786,7 @@ module GoodData
       def to_maql_create
         maql = "# Create the '#{self.title}' data set\n"
         maql += "CREATE DATASET {#{self.identifier}} VISUAL (TITLE \"#{self.title}\");\n\n"
-        [ attributes, facts, { 1 => @connection_point } ].each do |objects|
+        [ attributes, facts, { 1 => @anchor } ].each do |objects|
           objects.values.each do |obj|
             maql += "# Create '#{obj.title}' and add it to the '#{self.title}' data set.\n"
             maql += obj.to_maql_create
@@ -709,26 +829,37 @@ module GoodData
       # 
       def to_manifest(mode="FULL")
         {
-          'dataSetSLIManifest' => {
-            'parts'   => fields.reduce([]) { |memo, f| val = f.to_manifest_part(mode); memo << val unless val.nil?; memo },
-            'dataSet' => self.identifier,
-            'file'    => 'data.csv', # should be configurable
-              'csvParams' => {
-              'quoteChar'     => '"',
-              'escapeChar'    => '"',
-              'separatorChar' => ',',
-              'endOfLine'     => "\n"
+            'dataSetSLIManifest' => {
+                'parts' => fields.reduce([]) { |memo, f| val = f.to_manifest_part(mode); memo << val unless val.nil?; memo },
+                'dataSet' => self.identifier,
+                'file' => 'data.csv', # should be configurable
+                'csvParams' => {
+                    'quoteChar' => '"',
+                    'escapeChar' => '"',
+                    'separatorChar' => ',',
+                    'endOfLine' => "\n"
+                }
             }
-          }
         }
       end
 
+      def to_wire_model
+        {
+          "dataset" => {
+                "identifier" => identifier,
+                "title" => title,
+                "anchor" => @anchor.to_wire_model,
+                "facts" => facts.map {|f| f.to_wire_model},
+                "attributes" => attributes.map {|a| a.to_wire_model},
+                "references" => references.map {|r| r.is_a?(DateReference) ? r.schema_ref : type_prefix + "." + r.schema_ref }}}
+      end
+
       private
 
       def add_attribute(column)
         attribute = Attribute.new column, self
         fields << attribute
-        add_to_hash(attributes, attribute)
+        attributes << attribute
         add_attribute_folder(attribute.folder)
         # folders[AttributeFolder.new(attribute.folder)] = 1 if attribute.folder
       end
@@ -742,7 +873,7 @@ module GoodData
       def add_fact(column)
         fact = Fact.new column, self
         fields << fact
-        add_to_hash(facts, fact)
+        facts << fact
         add_fact_folder(fact.folder)
         # folders[FactFolder.new(fact.folder)] = 1 if fact.folder
       end
@@ -762,24 +893,23 @@ module GoodData
       def add_reference(column)
         reference = Reference.new(column, self)
         fields << reference
-        add_to_hash(references, reference)
+        references << reference
       end
 
       def add_date(column)
         date = DateColumn.new column, self
         @fields << date
         date.parts.values.each { |p| @fields << p }
-        date.facts.each { |f| add_to_hash(self.facts, f) }
-        date.attributes.each { |a| add_to_hash(self.attributes, a) }
-        date.references.each {|r| add_to_hash(self.references, r)}
+        date.facts.each { |f| facts << f }
+        date.attributes.each { |a| attributes << a }
+        date.references.each {|r| references << r}
       end
 
-      def set_connection_point(column)
-        @connection_point = RecordsOf.new column, self
-        @fields << @connection_point
+      def set_anchor(column)
+        @anchor = Anchor.new column, self
+        @fields << @anchor
       end
 
-      def add_to_hash(hash, obj); hash[obj.identifier] = obj; end
     end
 
     ##
@@ -792,10 +922,10 @@ module GoodData
       def initialize(hash, schema)
         super()
         raise ArgumentError.new("Schema must be provided, got #{schema.class}") unless schema.is_a? Schema
-        @name    = hash[:name] || raise("Data set fields must have their names defined")
-        @title   = hash[:title] || hash[:name].humanize
-        @folder  = hash[:folder]
-        @schema  = schema
+        @name = hash[:name] || raise("Data set fields must have their names defined")
+        @title = hash[:title] || hash[:name].humanize
+        @folder = hash[:folder]
+        @schema = schema
       end
 
       ##
@@ -838,13 +968,19 @@ module GoodData
     # GoodData attribute abstraction
     #
     class Attribute < Column
-      attr_reader :primary_label
+      attr_reader :primary_label, :labels
 
-      def type_prefix ; ATTRIBUTE_PREFIX ; end
-      def folder_prefix; ATTRIBUTE_FOLDER_PREFIX; end
+      def type_prefix;
+        ATTRIBUTE_PREFIX;
+      end
+
+      def folder_prefix;
+        ATTRIBUTE_FOLDER_PREFIX;
+      end
 
       def initialize(hash, schema)
         super hash, schema
+        @labels = []
         @primary_label = Label.new hash, self, schema
       end
 
@@ -852,7 +988,9 @@ module GoodData
         @table ||= "d_" + @schema.name + "_" + name
       end
 
-      def key ; "#{@name}#{FK_SUFFIX}" ; end
+      def key;
+        "#{@name}#{FK_SUFFIX}";
+      end
 
       def to_maql_create
         maql = "CREATE ATTRIBUTE {#{identifier}} VISUAL (#{visual})" \
@@ -863,12 +1001,31 @@ module GoodData
 
       def to_manifest_part(mode)
         {
-          'referenceKey' => 1,
-          'populates' => [ @primary_label.identifier ],
-          'mode' => mode,
-          'columnName' => name
+            'referenceKey' => 1,
+            'populates' => [@primary_label.identifier],
+            'mode' => mode,
+            'columnName' => name
+        }
+      end
+
+      def to_wire_model
+        {
+            "attribute" => {
+                "identifier" => identifier,
+                "title" => title,
+                "labels" => labels.map do |l|
+                  {
+                    "label" => {
+                        "identifier" => l.identifier,
+                        "title" => l.title,
+                        "type" => "GDC.text"
+                    }
+                  }
+                end
+            }
         }
       end
+
     end
 
     ##
@@ -877,12 +1034,17 @@ module GoodData
     # field
     #
     class Label < Column
+
+      attr_accessor :attribute
+
       def type_prefix ; 'label' ; end
 
       # def initialize(hash, schema)
       def initialize(hash, attribute, schema)
         super hash, schema
-        @attribute = attribute || schema.fields.find {|field| field.name === hash[:reference]}
+        attribute = attribute.nil? ? schema.fields.find {|field| field.name === hash[:reference]} : attribute
+        @attribute = attribute
+        attribute.labels << self
       end
 
       def to_maql_create
@@ -893,9 +1055,9 @@ module GoodData
 
       def to_manifest_part(mode)
         {
-          'populates'     => [ identifier ],
-          'mode'          => mode,
-          'columnName'    => name
+            'populates' => [identifier],
+            'mode' => mode,
+            'columnName' => name
         }
       end
 
@@ -904,6 +1066,7 @@ module GoodData
       end
 
       alias :inspect_orig :inspect
+
       def inspect
         inspect_orig.sub(/>$/, " @attribute=" + @attribute.to_s.sub(/>$/, " @name=#{@attribute.name}") + '>')
       end
@@ -913,15 +1076,14 @@ module GoodData
     # A GoodData attribute that represents a data set's connection point or a data set
     # without a connection point
     #
-    class RecordsOf < Attribute
+    class Anchor < Attribute
       def initialize(column, schema)
         if column then
           super
         else
-          @name = 'id'
-          @title = "Records of #{schema.name}"
-          @folder = nil
-          @schema = schema
+          super({:type => "anchor", :name => 'id'}, schema)
+          @labels = []
+          @primary_label = nil
         end
       end
 
@@ -938,15 +1100,24 @@ module GoodData
         end
         maql
       end
+
     end
 
     ##
     # GoodData fact abstraction
     #
     class Fact < Column
-      def type_prefix ; FACT_PREFIX ; end
-      def column_prefix ; FACT_COLUMN_PREFIX ; end
-      def folder_prefix; FACT_FOLDER_PREFIX; end
+      def type_prefix;
+        FACT_PREFIX;
+      end
+
+      def column_prefix;
+        FACT_COLUMN_PREFIX;
+      end
+
+      def folder_prefix;
+        FACT_FOLDER_PREFIX;
+      end
 
       def table
         @schema.table
@@ -963,9 +1134,18 @@ module GoodData
 
       def to_manifest_part(mode)
         {
-          'populates'  => [ identifier ],
-          'mode'       => mode,
-          'columnName' => name
+            'populates' => [identifier],
+            'mode' => mode,
+            'columnName' => name
+        }
+      end
+
+      def to_wire_model
+        {
+          "fact" => {
+            "identifier" => identifier,
+            "title" => title
+          }
         }
       end
     end
@@ -974,13 +1154,16 @@ module GoodData
     # Reference to another data set
     #
     class Reference < Column
+
+      attr_accessor :reference, :schema_ref
+
       def initialize(column, schema)
         super column, schema
         # pp column
-        @name       = column[:name]
-        @reference  = column[:reference]
+        @name = column[:name]
+        @reference = column[:reference]
         @schema_ref = column[:dataset]
-        @schema     = schema
+        @schema = schema
       end
 
       ##
@@ -992,7 +1175,9 @@ module GoodData
         @identifier ||= "#{ATTRIBUTE_PREFIX}.#{@schema_ref}.#{@reference}"
       end
 
-      def key ; "#{@name}_id" ; end
+      def key;
+        "#{@name}_id";
+      end
 
       def label_column
         "#{LABEL_PREFIX}.#{@schema_ref}.#{@reference}"
@@ -1008,52 +1193,14 @@ module GoodData
 
       def to_manifest_part(mode)
         {
-          'populates'     => [ label_column ],
-          'mode'          => mode,
-          'columnName'    => name,
-          'referenceKey'  => 1
+            'populates' => [label_column],
+            'mode' => mode,
+            'columnName' => name,
+            'referenceKey' => 1
         }
       end
     end
 
-    ##
-    # Fact representation of a date.
-    #
-    # class DateFact < Fact
-    # 
-    #   attr_accessor :format, :output_format
-    # 
-    #   def initialize(column, schema)
-    #     super column, schema
-    #     @output_format = column["format"] || '("dd/MM/yyyy")'
-    #     @format = @output_format.gsub('yyyy', '%Y').gsub('MM', '%m').gsub('dd', '%d')
-    #   end
-    # 
-    #   def column_prefix ; DATE_COLUMN_PREFIX ; end
-    #   def type_prefix ; DATE_FACT_PREFIX ; end
-    # 
-    #   def to_csv_header(row)
-    #     "#{name}_fact"
-    #   end
-    # 
-    #   def to_csv_data(headers, row)
-    #     val = row[name]
-    #     val.nil?() ? nil : (Date.strptime(val, format) - BEGINNING_OF_TIMES).to_i
-    #     rescue ArgumentError
-    #       raise "Value \"#{val}\" for column \"#{name}\" did not match the format: #{format}. " +
-    #         "Perhaps you need to add or change the \"format\" key in the data set configuration."
-    #   end
-    # 
-    #   def to_manifest_part(mode)
-    #     {
-    #       'populates'  => [ identifier ],
-    #       'mode'       => mode,
-    #       'columnName' => "#{name}_fact"
-    #     }
-    #   end
-    # 
-    # end
-
     ##
     # Date as a reference to a date dimension
     #
@@ -1074,11 +1221,11 @@ module GoodData
 
       def to_manifest_part(mode)
         {
-          'populates'     => [ "#{identifier}.#{DATE_ATTRIBUTE_DEFAULT_DISPLAY_FORM}" ],
-          'mode'          => mode,
-          'constraints'   => {"date" => output_format},
-          'columnName'    => name,
-          'referenceKey'  => 1
+            'populates' => ["#{identifier}.#{DATE_ATTRIBUTE_DEFAULT_DISPLAY_FORM}"],
+            'mode' => mode,
+            'constraints' => {"date" => output_format},
+            'columnName' => name,
+            'referenceKey' => 1
         }
       end
 
@@ -1090,21 +1237,22 @@ module GoodData
       #   # maql += "INCLUDE TEMPLATE \"#{urn}\" MODIFY (IDENTIFIER \"#{name}\", TITLE \"#{title || name}\");\n"
       #   maql += super_maql
       # end
-
     end
 
     ##
     # Date field that's not connected to a date dimension
     #
     class DateAttribute < Attribute
-      def key ; "#{DATE_COLUMN_PREFIX}#{super}" ; end
+      def key;
+        "#{DATE_COLUMN_PREFIX}#{super}";
+      end
 
       def to_manifest_part(mode)
         {
-          'populates'     => ['label.stuff.mmddyy'],
-          "format"        => "unknown",
-          "mode"          => mode,
-          "referenceKey"  => 1
+            'populates' => ['label.stuff.mmddyy'],
+            "format" => "unknown",
+            "mode" => mode,
+            "referenceKey" => 1
         }
       end
     end
@@ -1113,8 +1261,13 @@ module GoodData
     # Fact representation of a time of a day
     #
     class TimeFact < Fact
-      def column_prefix ; TIME_COLUMN_PREFIX ; end
-      def type_prefix ; TIME_FACT_PREFIX ; end
+      def column_prefix;
+        TIME_COLUMN_PREFIX;
+      end
+
+      def type_prefix;
+        TIME_FACT_PREFIX;
+      end
     end
 
     ##
@@ -1128,9 +1281,17 @@ module GoodData
     # Time field that's not connected to a time-of-a-day dimension
     #
     class TimeAttribute < Attribute
-      def type_prefix ; TIME_ATTRIBUTE_PREFIX ; end
-      def key ; "#{TIME_COLUMN_PREFIX}#{super}" ; end
-      def table ; @table ||= "#{super}_tm" ; end
+      def type_prefix;
+        TIME_ATTRIBUTE_PREFIX;
+      end
+
+      def key;
+        "#{TIME_COLUMN_PREFIX}#{super}";
+      end
+
+      def table;
+        @table ||= "#{super}_tm";
+      end
     end
 
     ##
@@ -1143,7 +1304,7 @@ module GoodData
 
       def initialize(column, schema)
         super column, schema
-        @parts = {} ; @facts = [] ; @attributes = []; @references = []
+        @parts = {}; @facts = []; @attributes = []; @references = []
 
         # @facts << @parts[:date_fact] = DateFact.new(column, schema)
         if column[:dataset] then
@@ -1204,16 +1365,26 @@ module GoodData
     # GoodData attribute folder abstraction
     #
     class AttributeFolder < Folder
-      def type; "ATTRIBUTE"; end
-      def type_prefix; "dim"; end
+      def type;
+        "ATTRIBUTE";
+      end
+
+      def type_prefix;
+        "dim";
+      end
     end
 
     ##
     # GoodData fact folder abstraction
     #
     class FactFolder < Folder
-      def type; "FACT"; end
-      def type_prefix; "ffld"; end
+      def type;
+        "FACT";
+      end
+
+      def type_prefix;
+        "ffld";
+      end
     end
 
     class DateDimension < MdObject