gooddata-edge 0.6.27.edge

data/lib/gooddata/models/blueprint/project_builder.rb

@@ -0,0 +1,83 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'dashboard_builder'
+require_relative 'schema_builder'
+
+module GoodData
+  module Model
+    class ProjectBuilder
+      attr_accessor :data
+
+      class << self
+        def create_from_data(blueprint, title = 'Title')
+          pb = ProjectBuilder.new(title)
+          pb.data = blueprint.to_hash
+          pb
+        end
+
+        def create(title, _options = {}, &block)
+          pb = ProjectBuilder.new(title)
+          block.call(pb)
+          pb
+        end
+      end
+
+      def initialize(title, options = {})
+        @data = {}
+        @data[:title] = title
+        @data[:datasets] = []
+        @data[:date_dimensions] = []
+        @data.merge(options)
+      end
+
+      def add_date_dimension(id, options = {})
+        dimension = {
+          type: :date_dimension,
+          urn: options[:urn],
+          id: id,
+          title: options[:title]
+        }
+
+        data[:date_dimensions] << dimension
+      end
+
+      def add_dataset(id, options = {}, &block)
+        builder = GoodData::Model::SchemaBuilder.new(id, options)
+        block.call(builder) if block
+        fail 'Dataset has to have id defined' if id.blank?
+        datasets = data[:datasets]
+        if datasets.any? { |item| item[:id] == id }
+          ds = datasets.find { |item| item[:id] == id }
+          index = datasets.index(ds)
+          stuff = GoodData::Model.merge_dataset_columns(ds, builder.to_hash)
+          datasets.delete_at(index)
+          datasets.insert(index, stuff)
+        else
+          datasets << builder.to_hash
+        end
+      end
+
+      def to_json(options = {})
+        eliminate_empty = options[:eliminate_empty] || false
+
+        if eliminate_empty
+          JSON.pretty_generate(to_hash.reject { |_k, v| v.is_a?(Enumerable) && v.empty? })
+        else
+          JSON.pretty_generate(to_hash)
+        end
+      end
+
+      def to_blueprint
+        GoodData::Model::ProjectBlueprint.new(to_hash)
+      end
+
+      def to_hash
+        data
+      end
+    end
+  end
+end

data/lib/gooddata/models/blueprint/reference_field.rb

@@ -0,0 +1,43 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'blueprint_field'
+
+module GoodData
+  module Model
+    class ReferenceBlueprintField < BlueprintField
+      # Returns the schema that is referenced by this ref
+      #
+      # @return [GoodData::Model::SchemaBlueprint] the referencesd schema
+      def dataset
+        dataset_blueprint.find_dataset(reference, include_date_dimensions: true)
+      end
+
+      # Returns the string reference of the ref which is string Id of dataset that is referenced.
+      #
+      # @return [String] Id of the referenced dataset
+      def reference
+        data[:dataset]
+      end
+
+      # Validates the fields in the ref
+      #
+      # @return [Array] returns list of the errors represented by hash structures
+      def validate
+        validate_presence_of(:dataset).map do |e|
+          { type: :error, message: "Field \"#{e}\" is not defined or empty for reference \"#{data}\"" }
+        end
+      end
+
+      # Returns the md object in associated project or throws error if not present
+      #
+      # @return [GoodData::MdObject] md object that is represented in the blueprint
+      def in_project(_project)
+        fail NotImplementedError, 'Reference does not have representation as an object in datamart'
+      end
+    end
+  end
+end

data/lib/gooddata/models/blueprint/schema_blueprint.rb

@@ -0,0 +1,160 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  module Model
+    class SchemaBlueprint
+      attr_accessor :data, :project_blueprint
+
+      def initialize(dim, blueprint)
+        @data = dim
+        @project_blueprint = blueprint
+      end
+
+      # Returns true if anchor is present. Currently returns always true. Probably good to remove
+      #
+      # @return [Boolean] is anchor on schema?l70
+      def anchor?
+        true
+      end
+
+      # Returns anchor
+      #
+      # @return [GoodData::Model::AnchorBlueprintField] anchor on schema
+      def anchor
+        nil
+      end
+
+      # Returns list of all references defined on the schema.
+      #
+      # @return [Array<GoodData::Model::ReferenceBlueprintField>] refs on schema
+      def references
+        []
+      end
+
+      # Returns list of all facts defined on the schema.
+      #
+      # @return [Array<GoodData::Model::FactBlueprintField>] facts on schema
+      def facts(_id = :all, _options = {})
+        []
+      end
+
+      # Returns list of all attributes defined on the schema.
+      #
+      # @return [Array<GoodData::Model::AttributeBlueprintField>] attributes on schema
+      def attributes(_id = :all, _options = {})
+        []
+      end
+
+      # Returns list of all labels defined on the schema.
+      #
+      # @return [Array<GoodData::Model::LabelBlueprintField>] labels on schema
+      def labels(_id = :all, _options = {})
+        []
+      end
+
+      # Returns list of all fields defined on the schema.
+      #
+      # @return [Array<GoodData::Model::BlueprintField>] all fields on schema
+      def fields
+        []
+      end
+
+      # Returns list of attributes and anchor.
+      #
+      # @return [Array<GoodData::Model::AnchorBlueprintField | GoodData::Model::AttributeBlueprintField>]
+      def attributes_and_anchors
+        []
+      end
+
+      # Returns list of attributes that can break facts in a given dataset.
+      # This basically means that it is giving you all attributes from this
+      # dataset and datasets that are referenced by given dataset transitively.
+      # Includes only anchors that have labels.
+      #
+      # @return [Array<GoodData::Model::BlueprintField>]
+      def broken_by
+        attrs = attributes_and_anchors.reject { |a| a.labels.empty? }
+        attrs + references.map(&:dataset).flat_map(&:broken_by)
+      end
+
+      # Returns list of attributes that are broken by attributes in this dataset. This means
+      # all anchors and attributes from this dataset and the ones that are referenced by any
+      # dataset. It works transitively. Includes only anchors that have labels.
+      #
+      # @return [Array<GoodData::Model::AnchorBlueprintField | GoodData::Model::AttributeBlueprintField>]
+      def breaks
+        attrs = attributes_and_anchors.reject { |a| a.labels.empty? }
+        referenced_by.empty? ? attrs : attrs + referenced_by.flat_map(&:breaks)
+      end
+
+      # Relays request on finding a dataset in the associated project blueprint. Used by reference fields
+      #
+      # @param dataset [String] Name of a dataset
+      # @param options [Hash] additional options. See ProjectBlueprint form more
+      # @return [GoodData::Model::DatasetBlueprint] returns matching dataset or throws an error
+      def find_dataset(dataset, options = {})
+        project_blueprint.find_dataset(dataset, options)
+      end
+
+      # Returns id of the schema
+      #
+      # @return [String] returns id
+      def id
+        data[:id]
+      end
+
+      # Returns dataset that are referencing this dataset (directly through references not transitively).
+      #
+      # @return [Array<GoodData::Model::SchemaBlueprint>] returns id
+      def referenced_by
+        @project_blueprint.referencing(self)
+      end
+
+      # Returns dataset that are referenced by this dataset (directly through references not transitively).
+      #
+      # @return [Array<GoodData::Model::SchemaBlueprint>] returns id
+      def referencing
+        references.map(&:dataset)
+      end
+
+      # Returns title of the dataset. If it is not set up. It is generated for you
+      # based on the name which is titleized
+      #
+      # @return [String]
+      def title
+        data[:title] || GoodData::Helpers.titleize(data[:id])
+      end
+
+      # Validates the blueprint and returns true if model is valid. False otherwise.
+      #
+      # @return [Boolean] is model valid?
+      def valid?
+        validate.empty?
+      end
+
+      # Validates the blueprint and returns array of errors.
+      #
+      # @return [Array<Hash>] returns array of errors or empty array
+      def validate
+        fields.flat_map(&:validate)
+      end
+
+      # Compares two blueprints. This is done by comapring the hash represenatation.
+      # It has to be exacty identical including the order of the columns
+      #
+      # @param name [GoodData::Model::DatasetBlueprint] Name of a field
+      # @return [Boolean] matching fields
+      def ==(other)
+        to_hash == other.to_hash
+      end
+
+      def to_hash
+        @data
+      end
+    end
+  end
+end

data/lib/gooddata/models/blueprint/schema_builder.rb

@@ -0,0 +1,89 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  module Model
+    class SchemaBuilder
+      attr_accessor :data
+
+      class << self
+        def create_from_data(blueprint)
+          sc = SchemaBuilder.new
+          sc.data = blueprint.to_hash
+          sc
+        end
+
+        def create(id, options = {}, &block)
+          pb = SchemaBuilder.new(id, options)
+          block.call(pb)
+          pb
+        end
+      end
+
+      def initialize(id = nil, options = {})
+        @data = {
+          id: id,
+          type: :dataset,
+          columns: []
+        }.merge(options)
+      end
+
+      def name
+        data[:name]
+      end
+
+      def columns
+        data[:columns]
+      end
+
+      def add_column(column_def)
+        columns.push(column_def)
+        self
+      end
+
+      def add_anchor(id, options = {})
+        add_column({ type: :anchor, id: id }.merge(options))
+        self
+      end
+
+      def add_attribute(id, options = {})
+        add_column({ type: :attribute, id: id }.merge(options))
+        self
+      end
+
+      def add_fact(id, options = {})
+        data = { type: :fact, id: id }.merge(options)
+        add_column(data)
+        self
+      end
+
+      def add_label(id, options = {})
+        add_column({ type: :label, id: id }.merge(options))
+        self
+      end
+
+      def add_date(dataset_id, options = {})
+        add_column({ type: :date, dataset: dataset_id, format: GoodData::Model::DEFAULT_DATE_FORMAT }.merge(options))
+      end
+
+      def add_reference(dataset, options = {})
+        add_column({ type: :reference, dataset: dataset }.merge(options))
+      end
+
+      def to_json
+        JSON.pretty_generate(to_hash)
+      end
+
+      def to_hash
+        data
+      end
+
+      def to_blueprint
+        GoodData::Model::ProjectBlueprint.new(datasets: [to_hash])
+      end
+    end
+  end
+end

data/lib/gooddata/models/blueprint/to_manifest.rb

@@ -0,0 +1,181 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2015 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module GoodData
+  module Model
+    module ToManifest
+      # Converts attribute or anchor to manifest
+      #
+      # @param project [GoodData::Model::ProjectBlueprint] Project blueprint
+      # @param dataset [GoodData::Model::SchemaBlueprint] Dataset blueprint
+      # @param attribute [Hash] Attribute or Anchor
+      # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
+      # @return [Hash] Manifest for a particular reference
+      def self.attribute_to_manifest(_project, dataset, a, mode)
+        labels = DatasetBlueprint.labels_for_attribute(dataset, a)
+        [{
+          'referenceKey' => 1,
+          'populates' => [labels.first[:id]],
+          'mode' => mode,
+          'columnName' => labels.first[:column_name] || labels.first[:id]
+        }]
+      end
+
+      # Sets the active project
+      #
+      # @param project A project identifier
+      #
+      # ### Examples
+      #
+      # The following calls are equivalent
+      #
+      def self.column_to_manifest(project, dataset, c, mode)
+        case c[:type].to_sym
+        when :label
+          label_to_manifest(project, dataset, c, mode)
+        when :fact
+          fact_to_manifest(project, dataset, c, mode)
+        when :date_fact
+          fact_to_manifest(project, dataset, c, mode)
+        when :reference
+          reference_to_manifest(project, dataset, c, mode)
+        when :date
+          date_ref_to_manifest(project, dataset, c, mode)
+        else
+          []
+        end
+      end
+
+      # Converts dataset into manifest. Since for manifest of a dataset you need to
+      # have access to the whole project blueprint it requires both project and
+      # dataset blueprints. It generates the manifest for blueprint and then selects
+      # only the one for particular dataset
+      #
+      # @param project [GoodData::Model::ProjectBlueprint] Project blueprint
+      # @param dataset [GoodData::Model::SchemaBlueprint] Dataset blueprint
+      # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
+      # @return [Hash] Manifest for a particular dataset
+      def self.dataset_to_manifest(project, dataset, mode = 'FULL')
+        dataset = dataset.is_a?(String) ? Model::ProjectBlueprint.find_dataset(project, dataset) : dataset
+        dataset = dataset.to_hash
+        res = Model::ProjectBlueprint.datasets(project).zip(to_manifest(project, mode)).find do |ds|
+          ds.first == dataset
+        end
+        res[1]
+      end
+
+      # Converts data reference to manifest
+      #
+      # @param project [GoodData::Model::ProjectBlueprint] Project blueprint
+      # @param dataset [GoodData::Model::SchemaBlueprint] Dataset blueprint
+      # @param reference [Hash] Reference
+      # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
+      # @return [Hash] Manifest for a particular date reference
+      def self.date_ref_to_manifest(project, _dataset, reference, mode)
+        referenced_dataset = ProjectBlueprint.find_date_dimension(project, reference[:dataset])
+        ref = "#{referenced_dataset[:id]}.date.mdyy"
+        format = reference[:format] || GoodData::Model::DEFAULT_DATE_FORMAT
+        GoodData.logger.info("Using date format \"#{format}\" for referencing attribute \"#{ref}\" of date dimension \"#{referenced_dataset[:id]}\"")
+        [{
+          'populates' => [ref],
+          'mode' => mode,
+          'constraints' => { 'date' => format },
+          'columnName' => reference[:column_name] || reference[:dataset],
+          'referenceKey' => 1
+        }]
+      end
+
+      # Converts fact to manifest
+      #
+      # @param project [GoodData::Model::ProjectBlueprint] Project blueprint
+      # @param dataset [GoodData::Model::SchemaBlueprint] Dataset blueprint
+      # @param fact [Hash] Fact
+      # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
+      # @return [Hash] Manifest for a particular fact
+      def self.fact_to_manifest(_project, _dataset, fact, mode)
+        [{
+          'populates' => [fact[:id]],
+          'mode' => mode,
+          'columnName' => fact[:column_name] || fact[:id]
+        }]
+      end
+
+      # Generates safe name for upload
+      # @param dataset_path [String] Input name
+      # @return [String] Generated upload filename
+      def self.generate_upload_filename(dataset_path)
+        sanitized_name = dataset_path.gsub(/[^0-9a-z]/i, '_')
+        # ts = DateTime.now.strftime('%Y%m%d%H%M%S%6N')
+        # "#{sanitized_name}-#{ts}.csv"
+        "#{sanitized_name}.csv"
+      end
+
+      # Converts label to manifest
+      #
+      # @param project [GoodData::Model::ProjectBlueprint] Project blueprint
+      # @param dataset [GoodData::Model::SchemaBlueprint] Dataset blueprint
+      # @param label [Hash] Label
+      # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
+      # @return [Hash] Manifest for a particular label
+      def self.label_to_manifest(_project, dataset, label, mode)
+        a = DatasetBlueprint.attribute_for_label(dataset, label)
+        labels = DatasetBlueprint.labels_for_attribute(dataset, a)
+
+        label = {}.tap do |l|
+          l['referenceKey'] = 1 if labels.first == label
+          l['populates'] = [label[:id]]
+          l['mode'] = mode
+          l['columnName'] = label[:column_name] || label[:id]
+        end
+        [label]
+      end
+
+      # The entry function of the module. Converts the ProjectBlueprint to manifest
+      # to be used with SLI (GD loading interface).
+      #
+      # @param project [GoodData::Model::ProjectBlueprint] Project blueprint
+      # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
+      # @return [Hash] Manifest for a particular project
+      def self.to_manifest(project, mode = 'FULL')
+        ProjectBlueprint.datasets(project.to_hash).map do |dataset|
+          columns = GoodData::Model::DatasetBlueprint.columns(dataset)
+          {
+            'dataSetSLIManifest' => {
+              'parts' => columns.mapcat { |c| column_to_manifest(project, dataset, c, mode) },
+              'dataSet' => dataset[:id],
+              'file' => ToManifest.generate_upload_filename(dataset[:id]), # should be configurable
+              'csvParams' => {
+                'quoteChar' => '"',
+                'escapeChar' => '"',
+                'separatorChar' => ',',
+                'endOfLine' => "\n"
+              }
+            }
+          }
+        end
+      end
+
+      # Converts reference to manifest
+      #
+      # @param project [GoodData::Model::ProjectBlueprint] Project blueprint
+      # @param dataset [GoodData::Model::SchemaBlueprint] Dataset blueprint
+      # @param reference [Hash] Reference
+      # @param mode [String] Mode of the load. Either FULL or INCREMENTAL
+      # @return [Hash] Manifest for a particular reference
+      def self.reference_to_manifest(project, _dataset, reference, mode)
+        referenced_dataset = ProjectBlueprint.find_dataset(project, reference[:dataset])
+        anchor = DatasetBlueprint.anchor(referenced_dataset)
+        label = DatasetBlueprint.labels_for_attribute(referenced_dataset, anchor).first
+        [{
+          'populates' => [label[:id]],
+          'mode' => mode,
+          'columnName' => reference[:column_name] || reference[:dataset],
+          'referenceKey' => 1
+        }]
+      end
+    end
+  end
+end