gooddata 0.6.0 → 0.6.2

data/lib/gooddata/models/project_blueprint.rb

@@ -0,0 +1,221 @@
+# encoding: UTF-8
+
+module GoodData
+  module Model
+    class ProjectBlueprint
+      attr_accessor :data
+
+      def self.from_json(spec)
+        if spec.is_a?(String)
+          ProjectBlueprint.new(MultiJson.load(File.read(spec), :symbolize_keys => true))
+        else
+          ProjectBlueprint.new(spec)
+        end
+      end
+
+      def change(&block)
+        builder = ProjectBuilder.create_from_data(self)
+        block.call(builder)
+        @data = builder.to_hash
+        self
+      end
+
+      def datasets
+        data[:datasets].map { |d| DatasetBlueprint.new(d) }
+      end
+
+      def add_dataset(a_dataset, index = nil)
+        if index.nil? || index > datasets.length
+          data[:datasets] << a_dataset.to_hash
+        else
+          data[:datasets].insert(index, a_dataset.to_hash)
+        end
+      end
+
+      def remove_dataset(dataset_name)
+        dataset = dataset_name.is_a?(String) ? find_dataset(dataset_name) : dataset_name
+        index = data[:datasets].index(dataset)
+        data[:datasets].delete_at(index)
+      end
+
+      def date_dimensions
+        data[:date_dimensions]
+      end
+
+      def dataset?(name)
+        found = data[:datasets].find { |d| d[:name] == name }
+        found != nil
+      end
+
+      def find_dataset(name)
+        ds = data[:datasets].find { |d| d[:name] == name }
+        fail "Dataset #{name} could not be found" if ds.nil?
+        DatasetBlueprint.new(ds)
+      end
+
+      def initialize(init_data)
+        @data = init_data
+      end
+
+      def validate_references
+        if datasets.count == 1
+          []
+        else
+          x = datasets.reduce([]) { |a, e| e.anchor? ? a << [e.name, e.anchor[:name]] : a }
+          refs = datasets.reduce([]) do |a, e|
+            a.concat(e.references)
+          end
+          refs.reduce([]) do |a, e|
+            x.include?([e[:dataset], e[:reference]]) ? a : a.concat([e])
+          end
+        end
+      end
+
+      def validate_labels_references
+        datasets.reduce([]) { |a, e| a.concat(e.validate_label_references) }
+      end
+
+      def validate_model
+        refs_errors = validate_references
+        labels_errors = validate_labels_references
+        refs_errors.concat(labels_errors)
+      end
+
+      def model_valid?
+        refs_errors = validate_references
+        labels_errors = validate_labels_references
+        errors = refs_errors.concat(labels_errors)
+        errors.empty? ? true : false
+      end
+
+      def referenced_by(dataset)
+        dataset = find_dataset(dataset) if dataset.is_a?(String)
+        dataset.references.map do |ds|
+          find_dataset(ds[:dataset])
+        end
+      end
+
+      def attributes
+        datasets.reduce([]) { |a, e| a.concat(e.attributes) }
+      end
+
+      def attributes_and_anchors
+        datasets.reduce([]) { |a, e| a.concat(e.attributes_and_anchors) }
+      end
+
+      def labels
+        datasets.reduce([]) { |a, e| a.concat(e.labels) }
+      end
+
+      def facts
+        datasets.reduce([]) { |a, e| a.concat(e.facts) }
+      end
+
+      def can_break(dataset)
+        dataset = find_dataset(dataset) if dataset.is_a?(String)
+        referenced_by(dataset).reduce([]) do |a, e|
+          e.attributes_and_anchors.each do |attr|
+            a.push([e, attr])
+          end
+          a
+        end
+      end
+
+      def find_star_centers
+        referenced = datasets.map { |d| referenced_by(d) }
+        referenced.flatten!
+        res = datasets.map(&:to_hash) - referenced.map(&:to_hash)
+        res.map { |d| DatasetBlueprint.new(d) }
+      end
+
+      def suggest_reports(options = {})
+        strategy = options[:strategy] || :stupid
+        case strategy
+        when :stupid
+          reports = suggest_metrics.reduce([]) do |a, e|
+            star, metrics = e
+            metrics.each { |m| m.save }
+            reports_stubs = metrics.map do |m|
+              breaks = can_break(star).map { |ds, aM| ds.identifier_for(aM) }
+              # [breaks.sample((breaks.length/10.0).ceil), m]
+              [breaks, m]
+            end
+            a.concat(reports_stubs)
+          end
+          reports.reduce([]) do |a, e|
+            attrs, metric = e
+
+            attrs.each do |attr|
+              a << GoodData::Report.create(
+                    :title => 'Fantastic report',
+                    :top => [attr],
+                    :left => metric)
+            end
+            a
+          end
+        end
+      end
+
+      def suggest_metrics
+        stars = find_star_centers
+        metrics = stars.map { |s| s.suggest_metrics }
+        stars.zip(metrics)
+      end
+
+      def merge!(a_blueprint)
+        temp_blueprint = merge(a_blueprint)
+        @data = temp_blueprint.data
+        self
+      end
+
+      def merge(a_blueprint)
+        temp_blueprint = dup
+        a_blueprint.datasets.each do |dataset|
+          if temp_blueprint.dataset?(dataset.name)
+            local_dataset = temp_blueprint.find_dataset(dataset.name)
+            index = temp_blueprint.datasets.index(local_dataset)
+            local_dataset.merge!(dataset)
+            temp_blueprint.remove_dataset(local_dataset.name)
+            temp_blueprint.add_dataset(local_dataset, index)
+          else
+            temp_blueprint.add_dataset(dataset.dup)
+          end
+        end
+        temp_blueprint
+      end
+
+      def dup
+        deep_copy = Marshal.load(Marshal.dump(data))
+        ProjectBlueprint.new(deep_copy)
+      end
+
+      def title
+        data[:title]
+      end
+
+      def to_wire_model
+        {
+          'diffRequest' => {
+            'targetModel' => {
+              'projectModel' => {
+                'datasets' => datasets.map { |d| d.to_wire_model },
+                'dateDimensions' => date_dimensions.map do |d|
+                  {
+                    'dateDimension' => {
+                      'name' => d[:name],
+                      'title' => d[:title] || d[:name].humanize
+                    }
+                  }
+                end
+              }
+            }
+          }
+        }
+      end
+
+      def to_hash
+        @data
+      end
+    end
+  end
+end

data/lib/gooddata/models/project_builder.rb

@@ -0,0 +1,136 @@
+# encoding: UTF-8
+
+require_relative 'dashboard_builder'
+require_relative 'schema_builder'
+
+module GoodData
+  module Model
+    class ProjectBuilder
+      attr_reader :title, :datasets, :reports, :metrics, :uploads, :users, :assert_report, :date_dimensions
+
+      class << self
+        def create_from_data(blueprint, title = 'Title')
+          pb = ProjectBuilder.new(title)
+          pb.data = blueprint.to_hash
+          pb
+        end
+
+        def create(title, options = {}, &block)
+          pb = ProjectBuilder.new(title)
+          block.call(pb)
+          pb
+        end
+      end
+
+      def initialize(title)
+        @title = title
+        @datasets = []
+        @reports = []
+        @assert_tests = []
+        @metrics = []
+        @uploads = []
+        @users = []
+        @dashboards = []
+        @date_dimensions = []
+      end
+
+      def add_date_dimension(name, options = {})
+        dimension = {
+          urn: options[:urn],
+          name: name,
+          title: options[:title]
+        }
+
+        @date_dimensions << dimension
+      end
+
+      def add_dataset(name, &block)
+        builder = GoodData::Model::SchemaBuilder.new(name)
+        block.call(builder)
+        if @datasets.any? { |item| item[:name] == name }
+          ds = @datasets.find { |item| item[:name] == name }
+          index = @datasets.index(ds)
+          stuff = GoodData::Model.merge_dataset_columns(ds, builder.to_hash)
+          @datasets.delete_at(index)
+          @datasets.insert(index, stuff)
+        else
+          @datasets << builder.to_hash
+        end
+      end
+
+      def add_report(title, options = {})
+        @reports << { :title => title }.merge(options)
+      end
+
+      def add_metric(title, options = {})
+        @metrics << { :title => title }.merge(options)
+      end
+
+      def add_dashboard(title, &block)
+        db = DashboardBuilder.new(title)
+        block.call(db)
+        @dashboards << db.to_hash
+      end
+
+      def load_metrics(file)
+        new_metrics = MultiJson.load(open(file).read, :symbolize_keys => true)
+        @metrics += new_metrics
+      end
+
+      def load_datasets(file)
+        new_metrics = MultiJson.load(open(file).read, :symbolize_keys => true)
+        @datasets += new_metrics
+      end
+
+      def assert_report(report, result)
+        @assert_tests << { :report => report, :result => result }
+      end
+
+      def upload(data, options = {})
+        mode = options[:mode] || 'FULL'
+        dataset = options[:dataset]
+        @uploads << {
+          :source => data,
+          :mode => mode,
+          :dataset => dataset
+        }
+      end
+
+      def add_users(users)
+        @users << users
+      end
+
+      def to_json(options = {})
+        eliminate_empty = options[:eliminate_empty] || false
+
+        if eliminate_empty
+          JSON.pretty_generate(to_hash.reject { |k, v| v.is_a?(Enumerable) && v.empty? })
+        else
+          JSON.pretty_generate(to_hash)
+        end
+      end
+
+      def to_blueprint
+        GoodData::Model::ProjectBlueprint.new(to_hash)
+      end
+
+      def to_hash
+        {
+          :title => @title,
+          :datasets => @datasets,
+          :uploads => @uploads,
+          :dashboards => @dashboards,
+          :metrics => @metrics,
+          :reports => @reports,
+          :users => @users,
+          :assert_tests => @assert_tests,
+          :date_dimensions => @date_dimensions
+        }
+      end
+
+      def find_dataset(name)
+        datasets.find { |d| d.name == name }
+      end
+    end
+  end
+end

data/lib/gooddata/models/project_creator.rb

@@ -0,0 +1,138 @@
+# encoding: UTF-8
+
+require_relative 'project'
+require_relative 'project_blueprint'
+
+require 'open-uri'
+
+module GoodData
+  module Model
+    class ProjectCreator
+      class << self
+        def migrate(options = {})
+          spec = options[:spec] || fail('You need to provide spec for migration')
+          spec = spec.to_hash
+
+          token = options[:token]
+          project = options[:project] || GoodData::Project.create(:title => spec[:title], :auth_token => token)
+          fail('You need to specify token for project creation') if token.nil? && project.nil?
+
+          begin
+            GoodData.with_project(project) do |p|
+              # migrate_date_dimensions(p, spec[:date_dimensions] || [])
+              migrate_datasets(p, spec)
+              load(p, spec)
+              migrate_metrics(p, spec[:metrics] || [])
+              migrate_reports(p, spec[:reports] || [])
+              migrate_dashboards(p, spec[:dashboards] || [])
+              migrate_users(p, spec[:users] || [])
+              execute_tests(p, spec[:assert_tests] || [])
+              p
+            end
+          end
+        end
+
+        def migrate_date_dimensions(project, spec)
+          spec.each do |dd|
+            Model.add_schema(DateDimension.new(dd), project)
+          end
+        end
+
+        def migrate_datasets(project, spec)
+          bp = ProjectBlueprint.new(spec)
+          # schema = Schema.load(schema) unless schema.respond_to?(:to_maql_create)
+          # project = GoodData.project unless project
+          uri = "/gdc/projects/#{GoodData.project.pid}/model/diff"
+          result = GoodData.post(uri, bp.to_wire_model)
+          link = result['asyncTask']['link']['poll']
+          response = GoodData.get(link, :process => false)
+          # pp response
+          while response.code != 200
+            sleep 1
+            GoodData.connection.retryable(:tries => 3, :on => RestClient::InternalServerError) do
+              sleep 1
+              response = GoodData.get(link, :process => false)
+              # pp response
+            end
+          end
+          response = GoodData.get(link)
+          ldm_links = GoodData.get project.md[LDM_CTG]
+          ldm_uri = Links.new(ldm_links)[LDM_MANAGE_CTG]
+          chunks = pick_correct_chunks(response['projectModelDiff']['updateScripts'])
+          chunks['updateScript']['maqlDdlChunks'].each do |chunk|
+            response = GoodData.post ldm_uri, 'manage' => { 'maql' => chunk }
+            polling_url = response['entries'].first['link']
+
+            polling_result = GoodData.get(polling_url)
+            while polling_result['wTaskStatus']['status'] == 'RUNNING'
+              sleep(3)
+              polling_result = GoodData.get(polling_url)
+            end
+            fail 'Creating dataset failed' if polling_result['wTaskStatus']['status'] == 'ERROR'
+
+          end
+
+          bp.datasets.each do |ds|
+            schema = ds.to_schema
+            GoodData::ProjectMetadata["manifest_#{schema.name}"] = schema.to_manifest.to_json
+          end
+        end
+
+        def migrate_reports(project, spec)
+          spec.each do |report|
+            project.add_report(report)
+          end
+        end
+
+        def migrate_dashboards(project, spec)
+          spec.each do |dash|
+            project.add_dashboard(dash)
+          end
+        end
+
+        def migrate_metrics(project, spec)
+          spec.each do |metric|
+            project.add_metric(metric)
+          end
+        end
+
+        def migrate_users(project, spec)
+          spec.each do |user|
+            puts "Would migrate user #{user}"
+            # project.add_user(user)
+          end
+        end
+
+        def load(project, spec)
+          if spec.key?(:uploads)
+            spec[:uploads].each do |load|
+              schema = GoodData::Model::Schema.new(spec[:datasets].find { |d| d[:name] == load[:dataset] })
+              project.upload(load[:source], schema, load[:mode])
+            end
+          end
+        end
+
+        def execute_tests(project, spec)
+          spec.each do |assert|
+            result = GoodData::ReportDefinition.execute(assert[:report])
+            fail "Test did not pass. Got #{result.table.inspect}, expected #{assert[:result].inspect}" if result.table != assert[:result]
+          end
+        end
+
+        def pick_correct_chunks(chunks)
+          # first is cascadeDrops, second is preserveData
+          rules = [
+            [false, true],
+            [false, false],
+            [true, true],
+            [true, false]
+          ]
+          stuff = chunks.select { |chunk| chunk['updateScript']['maqlDdlChunks'] }
+          rules.reduce(nil) do |a, e|
+            a || stuff.find { |chunk| e[0] == chunk['updateScript']['cascadeDrops'] && e[1] == chunk['updateScript']['preserveData'] }
+          end
+        end
+      end
+    end
+  end
+end