gd_bam 0.0.15 → 0.1.0

data/lib/utils/utils.rb

@@ -0,0 +1,66 @@
+module GoodData
+	module Bam
+		module Utils
+
+			PROJECT_GRAPHS_DIR = "graphs"
+	    PROJECT_METADATA_DIR = "metadata"
+	    PROJECT_DATA_DIR = "data"
+	    PARAM_FILE = 'params.json'  
+
+			def self.get_paths(base)
+	    	base = Pathname(base)
+				{
+					:home_path => base,
+					:graphs_path => base + PROJECT_GRAPHS_DIR,
+					:metadata_path => base + PROJECT_METADATA_DIR,
+					:data_path => base + PROJECT_DATA_DIR
+				}
+	    end
+
+	    def self.get_user_params(base)
+	    	base = Pathname(base)
+	    	param_path = base + PARAM_FILE
+	    	params = File.exist?(param_path) ? JSON.parse(File.read(param_path), :symbolize_names => true) : {}
+	    	{:additional_params => {}}.merge(params)
+	    end
+
+	    def self.render_template(template, params, options={})
+	      to_file = options[:to_file]
+	      root = options[:root] || GoodData::Bam::TEMPLATE_ROOT
+	      t = Pathname(template)
+	      output = Erubis::Eruby.new(File.read(root + t)).result(params)
+	      if to_file.nil?
+	        output
+	      else
+	        File.open(to_file, 'w') {|f| f.write(output)}
+	      end
+	    end
+
+      def self.setup_basic_project(paths, options={})
+        [:project_name, :project_pid].each do |param|
+          fail "Project #{param} needs to be specified for project template to be generated" if options[param].blank?	
+        end
+	      
+	      home = paths[:home_path]
+
+	      paths.values_at(:home_path, :graphs_path, :metadata_path, :data_path).each do |dir|
+	        FileUtils::mkdir_p dir
+	      end
+	      File.open(home + 'params.prm', 'w') do |f|
+          f << "TRASH=0"
+        end
+	      Utils::render_template("project.erb", options, :to_file => home + '.project')
+	      Utils::render_template("workspace.prm.erb", options, :to_file => home + 'workspace.prm')
+	    end
+
+      def self.should_backup_to_s3?(params)
+        keys = [:S3_SECRET_ACCESS_KEY, :S3_ACCESS_KEY_ID, :S3_BUCKETNAME]
+        if keys.any? {|key| params[:additional_params].has_key?(key) && params[:additional_params][key].blank?}
+          fail "You still have unfilled keys :S3_SECRET_ACCESS_KEY or :S3_ACCESS_KEY_ID or :S3_BUCKETNAME. Fill them in or delete them explicitely. If you do that your data will not be backed up."
+        end
+        keys.all? {|key| !params[:additional_params][key].blank?}
+      end
+
+		end
+	end
+end

data/templates/flow.rb.erb

@@ -1,11 +1,12 @@
-GoodData::CloverGenerator::DSL::flow("<%= flow_name %>") do |f|
+flow(:id => "<%= flow_name %>") do |f|
   tap(:id => "user")
 
-  graph("process_owner")
-  metadata("user") do |m|
-    m.remove("FirstName")
-    m.remove("LastName")
-    m.add(:name => "Name", :type => "string")
+  graph(:path => "process_owner.grf") do
+    metadata(:id => "user") do |m|
+      remove(:name => "FirstName")
+      remove(:name => "LastName")
+      add(:name => "Name")
+    end
   end
 
   sink(:id => "user")

data/templates/join_template.grf.erb

@@ -6,7 +6,7 @@
 <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/2_in.xml" id="Metadata2"/>
 <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/2_out.xml" id="Metadata3"/>
 <MetadataGroup id="ComponentGroup0" name="metadata"/>
-<Property fileURL="params.txt" id="GraphParameter14"/>
+<Property fileURL="params.prm" id="GraphParameter14"/>
 <Property fileURL="workspace.prm" id="GraphParameter0"/>
 <Dictionary/>
 </Global>

data/templates/reformat_template.grf.erb

@@ -4,7 +4,7 @@
 <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_in.xml" id="Metadata0"/>
 <Metadata fileURL="${PROJECT}/metadata/${FLOW}/${NAME}/1_out.xml" id="Metadata1"/>
 <MetadataGroup id="ComponentGroup0" name="metadata"/>
-<Property fileURL="params.txt" id="GraphParameter0"/>
+<Property fileURL="params.prm" id="GraphParameter0"/>
 <Property fileURL="workspace.prm" id="GraphParameter1"/>
 <Dictionary/>
 </Global>

data/templates/sink.json.erb

@@ -0,0 +1,28 @@
+{
+  "target" : "gooddata",
+  "id": "owner_dim",
+  "gd_name": "opp_owner",
+  "fields": [{
+    "type": "attribute",
+    "name": "id",
+    "meta": "Id"
+  }, {
+    "type": "label",
+    "for": "id",
+    "name": "name",
+    "meta": "Name"
+  }, {
+    "type": "label",
+    "for": "id",
+    "name": "url",
+    "meta": "Url"
+  }, {
+    "type": "attribute",
+    "name": "department",
+    "meta": "Department"
+  }, {
+    "type": "attribute",
+    "name": "region",
+    "meta": "Division"
+  }]
+}

data/templates/tap.json.erb

@@ -1,9 +1,7 @@
 {
-  "type" : "tap"
-  ,"source" : "salesforce"
-  ,"object" : "<%= tap_name %>"
-  ,"id" : "<%= tap_name %>"
-  ,"incremental" : false
+   "source" : "salesforce"
+  ,"object" : "User"
+  ,"id" : "user"
   ,"fields" : [
 		{
 			"name" : "Id"

data/templates/workspace.prm.erb

@@ -18,6 +18,10 @@ MODEL=${PROJECT}/model
 # You can find the ID of a project in the project context menu in Server Explorer.
 GDC_PROJECT_ID=<%= project_pid %>
 
+GD_LOGIN=<%= gd_login %>
+GD_PASSWORD=<%= gd_pass %>
+GDC_WEBDAV_HOST=secure-di.gooddata.com
+GRAPH_STARTED_TIMESTAMP=`to_string(date2long(today()))`L
 <% if additional_params %>
 	<% additional_params.each_pair do |key, val| %>
 <%= "#{key}=#{val}" %>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gd_bam
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-22 00:00:00.000000000 Z
+date: 2013-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -326,10 +326,54 @@ files:
 - bin/bam
 - lib/bam/version.rb
 - lib/bam.rb
-- lib/contract_checkers/contract_checkers.rb
-- lib/dsl/project_dsl.rb
+- lib/base/errors.rb
+- lib/base/flow.rb
+- lib/base/graph.rb
+- lib/base/metadata.rb
+- lib/base/project.rb
+- lib/base/repo.rb
+- lib/base/sink.rb
+- lib/base/step.rb
+- lib/base/tap.rb
+- lib/base/taps.rb
+- lib/cloud_connect/dsl/cc.rb
+- lib/cloud_connect/dsl/es_helpers.rb
+- lib/cloud_connect/dsl/helpers.rb
+- lib/cloud_connect/dsl/nodes.rb
+- lib/cloud_connect/dsl/sf_helpers.rb
+- lib/cloud_connect/dsl/structure_helpers.rb
+- lib/commands/commands.rb
+- lib/commands/deployment.rb
+- lib/commands/docs_commands.rb
+- lib/commands/gd_commands.rb
+- lib/commands/scaffold_commands.rb
+- lib/commands/sf_commands.rb
+- lib/commands/validators.rb
+- lib/compatibility.rb
+- lib/compiler/compiler.rb
+- lib/compiler/etl_visitor.rb
+- lib/dsl/dsl.rb
+- lib/generators/downloaders.rb
+- lib/generators/etl.rb
+- lib/generators/validators.rb
 - lib/graphs/docentize.grf
 - lib/graphs/dummy.grf
+- lib/graphs/goodsales_v2/docentize.grf
+- lib/graphs/goodsales_v2/dummy.grf
+- lib/graphs/goodsales_v2/load_history.grf
+- lib/graphs/goodsales_v2/process_account.grf
+- lib/graphs/goodsales_v2/process_activity.grf
+- lib/graphs/goodsales_v2/process_activity_dim.grf
+- lib/graphs/goodsales_v2/process_activity_owner.grf
+- lib/graphs/goodsales_v2/process_forecast.grf
+- lib/graphs/goodsales_v2/process_opp_records.grf
+- lib/graphs/goodsales_v2/process_opportunity.grf
+- lib/graphs/goodsales_v2/process_opportunity_line_item.grf
+- lib/graphs/goodsales_v2/process_opportunity_snapshot.grf
+- lib/graphs/goodsales_v2/process_owner.grf
+- lib/graphs/goodsales_v2/process_stage.grf
+- lib/graphs/goodsales_v2/process_stage_history.grf
+- lib/graphs/goodsales_v2/process_velocity_duration.grf
 - lib/graphs/load_history.grf
 - lib/graphs/process_account.grf
 - lib/graphs/process_activity.grf
@@ -346,17 +390,15 @@ files:
 - lib/graphs/process_velocity_duration.grf
 - lib/nodes/clover_gen.rb
 - lib/nodes/dependency.rb
-- lib/nodes/nodes.rb
-- lib/repo/1_config.json
 - lib/repository/repo.rb
 - lib/runtime.rb
-- templates/dataset.json.erb
+- lib/utils/utils.rb
 - templates/flow.rb.erb
 - templates/join_template.grf.erb
 - templates/params.json.erb
 - templates/project.erb
 - templates/reformat_template.grf.erb
-- templates/source.json.erb
+- templates/sink.json.erb
 - templates/tap.json.erb
 - templates/update_dataset.script.erb
 - templates/update_dataset_dry.script.erb

data/lib/contract_checkers/contract_checkers.rb

@@ -1,53 +0,0 @@
-module GoodData 
-	module BAM
-  	module ContractCheckers 
-
-  		include GoodData::CloverGenerator
-
-  		def self.generate(file)
-  			File.open(file, "w") do |file|
-
-	        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-	        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-	        builder.Graph({
-	          :name => "Contract Checker"
-	        }) do
-	          builder.Global do
-	            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-	          end
-
-	          builder.Phase(:number => 0) do
-
-	          end
-
-	          # phase = 0
-	          # subgraphs.each do |subgraph|
-	          #   builder.Phase(:number => phase+1) do
-	          #     id1 = get_id
-	          #     id2 = get_id
-	          #     ctl = "function integer generate() {$out.0.all = \"FLOW=#{subgraph[:flow]}\";return OK;}"
-	          #     build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
-	          #     build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata", :id => get_id()}))
-	          #     build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.txt", :outputFieldNames => "false", :quotedStrings => "false"}))
-	          #   end
-	          #   builder.Phase(:number => phase+2) do
-	              
-	          #     id1 = get_id
-	          #     id2 = get_id
-	          #     ctl = "function integer generate() {$out.0.all = \"NAME=#{subgraph[:name]}\";return OK;}"
-	          #     build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
-	          #     build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata", :id => get_id()}))
-	          #     build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.txt", :outputFieldNames => "false", :append => "true", :quotedStrings => "false"}))
-	          #   end  
-	            
-	          #   builder.Phase(:number => phase+3) do
-	          #     build_node2(builder, GoodData::CloverGenerator::Nodes.run_graph2({:guiName => subgraph[:name], :name => subgraph[:name], :id => subgraph[:flow], :graphName => subgraph[:file]}))
-	          #   end
-	          #   phase += 4
-	          # end
-	        end
-	      end
-  		end
-		end
-	end
-end

data/lib/dsl/project_dsl.rb

@@ -1,259 +0,0 @@
-require 'terminal-table'
-
-
-
-module GoodData 
-  module CloverGenerator
-    module DSL
-
-      class RemoveMetadataFieldError < RuntimeError
-
-        attr_reader :options, :metadata, :field
-
-        def initialize(message, options={})
-          super(message)
-          @options = options
-          @metadata = options[:metadata]
-          @field = options[:field]
-        end
-      end
-
-      class Metadata
-        
-        attr_accessor :metadata
-
-        def name
-          metadata[:name]
-        end
-        
-        def to_hash
-          metadata
-        end
-
-        def initialize(metadata)
-          @metadata = metadata
-        end
-        
-        def add(options={})
-          fail "You have to specify name at the metadata change. You specified #{what}" unless options.has_key?(:name)
-          position = options[:position] || 0
-          what = {
-            :name => options[:name],
-            :type => options[:type] || "string"
-          }
-
-          @metadata[:fields].insert(position - 1, what)
-          @metadata
-        end
-        
-        def remove(what)
-          fields = metadata[:fields]
-          fail RemoveMetadataFieldError.new("Specified column #{what} was not found", :field => what, :metadata => self) unless fields.detect {|f| f[:name] == what}
-          @metadata[:fields] = fields.find_all {|f| f[:name] != what}
-          @metadata
-        end
-
-        def change
-          yield(self)
-          self
-        end
-      end
-
-      class Flow
-
-        attr_accessor :steps, :name
-
-        def self.define(name="", &script)
-          # puts "Reading flow #{name}"
-          x = self.new
-          x.flow_name(name)
-          x.instance_eval(&script)
-          x
-        end
-
-        def initialize
-          @steps = []
-        end
-
-        def flow_name(name)
-          @name = name
-        end
-
-        def tap(options={}, &bl)
-          step({:type => :tap, :source_name => options[:id]})
-        end
-
-        def sink(options={}, &bl)
-          step(:type => :upload, :id => options[:id], &bl)
-        end
-
-        def graph(graph, &bl)
-          step(:graph => graph, :type => :user_provided, &bl)
-        end
-
-        def parallel(&bl)
-          
-        end
-
-        def step(options={}, &bl)
-          graph = options[:graph]
-          type = options[:type]
-
-          steps.push(options)
-          # puts "Running step #{graph}"
-        end
-
-        def metadata(name=nil,options={}, &bl)
-          steps.last[:metadata_block] = [] if steps.last[:metadata_block].nil?
-          steps.last[:metadata_block] << {:name => name, :block => bl, :out_as => options[:out_as]}
-        end
-
-
-      end
-
-      def self.flow(name="", &bl)
-        Flow.define(name, &bl)
-      end
-
-
-      class Project
-
-        attr_accessor :usecases, :name, :dims
-
-        def self.define(&script)
-          # print self
-          x = self.new
-          x.instance_eval(&script)
-          x
-        end
-
-        def initialize
-          @usecases = []
-        end
-
-        def project_name(name)
-          @name = name
-        end
-
-        def use_dims(dims)
-          @dims = dims
-        end
-  
-        def use_usecase(usecase)
-          @usecases << usecase
-        end
-        
-        
-  
-        def get_sources
-          configs = []
-          FileUtils.cd('./taps') do
-            Dir.glob('*.json').each do|f|
-              configs << JSON.parse(File.read(f), :symbolize_names => true)
-            end
-          end
-          configs
-        end
-
-        def print_sources(taps)
-          puts
-          puts "Printing sources"
-          puts "================"
-          puts
-          taps.each do |tap|
-            fail "Provided tap #{tap[:object]} does not seem to be tap" if tap[:type] != "tap"
-            if tap[:source] == "salesforce"
-              table = Terminal::Table.new(:title => "#{tap[:source]} => #{tap[:object]}", :style => {:width => 30}) do |t|
-                tap[:fields].each do |f|
-                  t << [f[:name], f[:name]]
-                end
-              end
-              puts table
-              puts
-            end
-          end
-
-        end
-
-        
-        def get_datasets
-          configs = []
-          FileUtils.cd('./sinks') do
-            Dir.glob('*.json').each do|f|
-              configs << JSON.parse(File.read(f), :symbolize_names => true)
-            end
-          end
-          configs
-        end
-
-        def compare_fields(sources, datasets)
-          a = sources.reduce([]) do |memo, source|
-            x = source[:object]
-            memo.concat(source[:fields].map {|f| [x, f[:name]]})
-            memo
-          end
-          
-          b = datasets.reduce([]) do |memo, source|
-            x = source[:name]
-            memo.concat(source[:fields].map {|f| [x, f[:name]]})
-            memo
-          end
-          result = (a | b) - (a & b)
-          if result.count > 0
-            puts "------------------"
-            puts "All fields not in"
-            puts "------------------"
-            result.each {|x| pp x}
-            fail "Some fields form source are not used"
-          end
-        end
-
-        def run(repo)
-          # puts "Running"
-
-          # puts "looking for dimension definitions"
-          # dims.each do |dim|
-            # puts "found #{dim}"
-          # end
-          
-          sources = get_sources
-          fail "You have no sources defined" if sources.empty?
-          # puts "Found #{sources.count} sources"
-          
-          
-          datasets = get_datasets
-          fail "You have no datasets defined" if datasets.empty?
-          # puts "Found #{datasets.count} sources"
-
-          # puts "Composing the tree"
-          you = GoodData::CloverGenerator::Dependency::N.new({
-            :name => name,
-            :type => "project",
-            :provides => [],
-            :requires => @usecases
-          })
-    
-          provided_dims = @dims.map do |dim_to_provide|
-            GoodData::CloverGenerator::Dependency::N.new({
-              :package => dim_to_provide.split("/").first,
-              :name => dim_to_provide.split("/").last,
-              :provides => [dim_to_provide.split("/").last],
-              :type => "dim",
-              :requires => []
-            })
-          end
-        
-          provided_dims.each {|x| repo << x}
-          # graph = resolve(repo, you)
-          # to_dot(graph)
-          v = GoodData::CloverGenerator::Dependency::Visitor.new
-        end
-
-      end
-
-      def self.project(&bl)
-        Project.define(&bl)
-      end
-    end
-  end
-end

data/lib/repo/1_config.json

@@ -1,8 +0,0 @@
-{
-	 "name" 		: "Quota attainment usecase"
-	,"type" 		: "usecase"
-	,"package" 		: "GoodSales"
-	,"version" 		: "0.0.1"
-	,"provides" 	: ["quota_attainment"]
-	,"requires" 	: ["GoodSales/opportunity", "GoodSales/user"]
-}

data/templates/dataset.json.erb

@@ -1,13 +0,0 @@
-{
-   "type"     : "dataset"
-  ,"id"       : "user"
-  ,"gd_name"  : "user"
-  ,"fields"   : [
-    {
-      "name" : "Id"
-    },
-    {
-      "name" : "Name"
-    }
-  ]
-}

data/templates/source.json.erb

@@ -1,22 +0,0 @@
-{
-   "source" : "salesforce"
-  ,"object" : "User"
-  ,"id"     : "user"
-  ,"fields" : [
-    {
-      "name" : "Id"
-    },
-    {
-      "name" : "FirstName"
-    },
-    {
-      "name" : "LastName"
-    },
-    {
-      "name" : "Region"
-    },
-    {
-      "name" : "Department"
-    }
-  ]
-}