gd_bam 0.0.15 → 0.1.0

data/lib/compiler/compiler.rb

@@ -0,0 +1,76 @@
+module GoodData 
+  module Bam
+    module Compiler
+
+      def self.compile_dsl(project)
+        fail "There are no flow. Nothing to work on." if project[:flows_definitions].empty?
+        compiled_flows = project[:flows_definitions].map do |flow_description|
+          Compiler.compile_flow(project, flow_description)
+        end
+        Project.create(project.merge({:flows => compiled_flows}))
+      end
+
+      def self.change_metadata(state, metadata_change_description)
+        metadata = state[metadata_change_description[:id]]
+
+        # enrich meta with everything from descriptor. There might be additional stuff so we want this to be part of metadata
+        enriched_meta = metadata.merge(metadata_change_description.reject {|k,v|  [:id, :steps, :type].include?(k)})
+
+        metadata_change_description[:steps].reduce(Metadata.create(enriched_meta)) do |meta_accumulator, step|
+          case step[:type]
+          when :field_add
+            Metadata.add_field(meta_accumulator, {:name => step[:name]})
+          when :field_remove
+            Metadata.remove_field(meta_accumulator, step[:name])
+          end
+        end
+      end
+      
+      def self.check_metadata_existence(metadata_store, id, flow_description)
+        unless metadata_store.has_key?(id)
+          fail "Metadata with id \"#{id}\" were not found. Check that you have them included in flow \"#{flow_description[:name]}\"."
+        end
+      end
+      
+      def self.compile_flow(project, flow_description)
+        flow = Flow.create({:name => flow_description[:name]})
+        # metadata_state = {}
+        out_port = nil
+
+        # this nil magic is done so we have visibility to one step back and forward. Currently not used much but it is useful for detectin input to sink
+        flow = [nil].concat(flow_description[:steps]).concat([nil]).each_cons(3).reduce(flow) do |flow_memo, steps|
+          previous_step, current_step, next_step = steps
+          id = current_step[:id] || current_step[:flow_id]
+
+          case current_step[:type]
+          when :dummy_tap
+            Flow.add_step(flow_memo, Tap.create(current_step.merge(:source => :dummy, :id => :none)))
+          when :tap
+            tap = Project.find_tap_by_id(project, id)
+            fail "Tap \"#{id}\" which was used in flow \"#{current_step[:flow_id]}\" is not defined" if tap.nil?
+            Flow.add_step(flow_memo, Tap.create(tap.merge(current_step)))
+          when :sink
+            sink = Project.find_sink_by_id(project, id)
+            fail "Sink \"#{id}\" not found" if sink.nil?
+            out_step = Step.find_output_step(previous_step[:steps])
+            Flow.add_step(flow_memo, Sink.create(sink.merge(current_step).merge({:in => out_step})))
+          when :graph
+            graph = Project.find_graph_by_path(project, current_step[:path])
+            fail "Graph on the path \"#{current_step[:path]}\" could not be found. This was specified in step \"#{current_step}\"" if graph.nil?
+            g = Graph.create(graph.merge(current_step))
+
+            if !Step.has_output_step?(g[:steps]) && g[:steps].count > 1
+              fail "You need to specify an output metadata."
+            end
+
+            Flow.add_step(flow_memo, Graph.create(g.merge({
+              :input_ports => g[:steps],
+              :output_port => Step.find_output_step(g[:steps])
+            })))
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/compiler/etl_visitor.rb

@@ -0,0 +1,165 @@
+module GoodData
+  module Bam
+    module Compiler
+      class EtlVisitor
+
+        def visit(node, state)
+          case node[:type]
+          when :project
+            FileUtils::cd(GoodData::Bam::Commands::ETL_HOME) do
+              flows = if state[:only]
+                node[:flows].find_all {|f| f[:name] == state[:only]}
+              else
+                node[:flows]
+              end
+              result = flows.reduce({
+                :flows => [],
+                :params => state
+              }) {|state, node| visit(node, state)}
+              GoodData::CloudConnect::Helpers::create_run_graph("graphs/main.grf", {
+                :subgraphs => result[:flows],
+                :flow => "main"
+              })
+            end
+          when :flow
+            visit_flow(node, state)
+          when :tap
+            case node[:direct]
+            when false
+              visit_incremental_tap(node, state)
+            else
+              case node[:source_type]
+              when :salesforce
+                visit_direct_sf_tap(node, state)
+              else
+                visit_direct_file_tap(node, state)
+              end
+            end
+          when :sink
+            case node[:target_type]
+            when :file, :web
+              visit_file_sink(node, state)
+            else
+              visit_gd_sink(node, state)
+            end
+          when :graph
+            visit_graph(node, state)
+          else
+            state
+          end
+        end
+
+        def visit_flow(flow, state)
+          puts "Visiting flow"
+          result = flow[:steps].reduce(state.merge({
+            :graphs => [],
+            :metadata => {}
+          })) {|state, node| visit(node, state)}
+            GoodData::CloudConnect::Helpers::create_run_graph("graphs/#{flow[:name]}_main.grf", {
+              :subgraphs => result[:graphs],
+              :flow => flow[:name]
+            })
+          state.merge(:flows => state[:flows].concat(["graphs/#{flow[:name]}_main.grf"]))
+        end
+  
+        def visit_graph(node, state)
+          puts "Visiting graph"
+          move_graphs = []
+          node[:input_ports].each_with_index do |input, i|
+            j = i+1
+            name = input[:id]
+            graph_path = "graphs/#{node[:flow_id]}_#{input[:id]}_move_in_#{j}.grf"
+            graph_name = "#{node[:flow_id]}_#{input[:id]}_move_in_#{j}.grf"
+            state = state.merge({:graphs => state[:graphs].concat([graph_path])})
+            GoodData::CloudConnect::Helpers::create_moving_graph(graph_path, {
+              :source => "${DATA}/#{name}.csv",
+              :target => "${DATA}/#{j}_in.csv",
+              :operation => "MOVE"
+            })
+
+            FileUtils.cp(Graph.get_path(node), (Pathname("graphs") + node[:path].basename).to_s)
+            changed_metadata = Compiler.change_metadata(state[:metadata], input)
+
+            metadata_path = "./metadata/#{node[:flow_id]}/#{Step.step_name(node)}/#{j}_in.xml"
+            GoodData::CloudConnect::Helpers::save_metadata(metadata_path, state[:metadata][name])
+            metadata_path = "./metadata/#{node[:flow_id]}/#{Step.step_name(node)}/#{j}_out.xml"
+            GoodData::CloudConnect::Helpers::save_metadata(metadata_path, changed_metadata)
+            state = state.merge({:metadata => state[:metadata].merge(changed_metadata[:id] => changed_metadata)})
+          end
+
+          state = state.merge({:graphs => state[:graphs].concat([(Pathname("graphs") + node[:path].basename).to_s])})
+
+          out_metadata = node[:output_port]
+          graph_path = "graphs/#{node[:flow_id]}_#{Step.step_name(node)}_move_out.grf"
+          state = state.merge({:graphs => state[:graphs].concat([graph_path])})
+          GoodData::CloudConnect::Helpers::create_moving_graph(graph_path, {
+            :source => "${DATA}/out.csv",
+            :target => "${DATA}/#{out_metadata[:id]}.csv",
+            :operation => "MOVE"
+          })
+          state
+        end
+
+        def visit_file_sink(node, state)
+          graph_path = Step.get_graph_path(node)
+
+          sink_fields = Metadata.get_source_metadata(node)[:fields].map {|f| f[:name]}
+          metadata_fields = state[:metadata][node[:in][:id]][:fields].map {|f| f[:name]}
+          fail "You have fields \"#{(sink_fields - metadata_fields).join(', ')}\" in sink \"#{node[:id]}\" that are not filled in by data" unless (sink_fields - metadata_fields).empty?
+
+          GoodData::Bam::Generators::Etl::create_file_uploading_graph(graph_path, node, node[:in][:id], state)
+          state.merge({
+            :graphs => state[:graphs].concat([graph_path])
+          })
+        end
+        
+
+        def visit_gd_sink(node, state)
+          graph_path = Step.get_graph_path(node)
+
+          sink_fields = Metadata.get_source_metadata(node)[:fields].map {|f| f[:name]}
+          metadata_fields = state[:metadata][node[:in][:id]][:fields].map {|f| f[:name]}
+          fail "You have fields \"#{(sink_fields - metadata_fields).join(', ')}\" in sink \"#{node[:id]}\" that are not filled in by data" unless (sink_fields - metadata_fields).empty?
+
+          GoodData::Bam::Generators::Etl::create_gd_uploading_graph(graph_path, node, node[:in][:id], state)
+          state.merge({
+            :graphs => state[:graphs].concat([graph_path])
+          })
+        end
+
+        def visit_direct_sf_tap(node, state)
+          puts "Visiting direct tap"
+          graph_path = Step.get_graph_path(node)
+          tap = Tap.prepare_for_sf_downloader(node)
+          GoodData::Bam::Generators::Etl::create_sf_downloading_graph(graph_path, tap, state)
+          state.merge({
+            :graphs => state[:graphs].concat([graph_path]),
+            :metadata => state[:metadata].merge({node[:id] => Metadata.get_target_metadata(node)})
+          })
+        end
+
+        def visit_direct_file_tap(node, state)
+          graph_path = Step.get_graph_path(node)
+          tap = Tap.prepare_for_sf_downloader(node)
+          GoodData::Bam::Generators::Etl::create_file_downloading_graph(graph_path, node, state)
+          state.merge({
+            :graphs => state[:graphs].concat([graph_path]),
+            :metadata => state[:metadata].merge({ node[:id] => Metadata.get_target_metadata(node) })
+          })
+        end
+
+        def visit_incremental_tap(node, state)
+          puts "Visiting Incremental tap"
+          # Get rid of all the shit. Timestamps, pull ids check for snapshots. And shove them downstream.
+          tap = Tap.prepare_for_es_downloader(node)
+          graph_path = Step.get_graph_path(tap)
+          GoodData::Bam::Generators::Etl::create_es_downloading_graph(graph_path, tap, state)
+          state.merge({
+            :graphs => state[:graphs].concat([graph_path]),
+            :metadata => state[:metadata].merge({tap[:id] => Metadata.get_target_metadata(tap)})
+          })
+        end
+      end
+    end
+  end
+end

data/lib/dsl/dsl.rb

@@ -0,0 +1,125 @@
+module GoodData 
+  module Bam
+    module DSL
+
+      def self.flow(data={}, &bl)
+        FlowDSL.new(data, &bl)
+      end
+
+      class FlowDSL
+
+        attr_reader :steps, :name
+
+        def initialize(data={}, &bl)
+          fail "Flows params needs a hash" unless data.is_a? Hash
+          name = data[:id]
+          fail "Flow needs to have a name" if name.blank?
+          @name = name
+          @steps = []
+          instance_eval(&bl)
+        end
+
+        def tap(data={})
+          fail "Params for tap need to be hash" unless data.is_a? Hash
+          @steps << {:type => :tap, :flow_id => @name}.merge(data)
+        end
+
+        def dummy_tap(data={})
+          fail "Params for tap need to be hash" unless data.is_a? Hash
+          @steps << {:type => :dummy_tap, :flow_id => @name}.merge(data)
+        end
+
+        def sink(data={})
+          fail "Params for tap need to be hash" unless data.is_a? Hash
+          @steps << {:type => :sink , :flow_id => @name}.merge(data)
+        end
+
+        def graph(data, &bl)
+          fail "Params for graph need to be hash" unless data.is_a? Hash
+          # @steps << {:type => :graph , :flow_id => @name}.merge(data)
+          if bl.nil?
+            @steps << {:type => :graph, :steps => [], :flow_id => @name}.merge(data)
+          else
+            metadata = GraphDSL.new(@name, &bl)
+            @steps << {:type => :graph, :steps => metadata.to_a, :flow_id => @name}.merge(data)
+          end
+        end
+
+        def to_hash
+          {
+            :name => @name,
+            :steps => @steps
+          }
+        end
+
+      end
+
+      class GraphDSL
+
+        def initialize(flow_name, &bl)
+          @name = flow_name
+          @steps = []
+          instance_eval(&bl)
+        end
+
+        def metadata(data, &bl)
+          fail "Params need to be hash" unless data.is_a? Hash
+          fail "Metadata definition should have at least \"id\" defined. You provided \"#{data}\"." unless data.has_key?(:id)
+          if bl.nil?
+            @steps << {:type => :graph_metadata, :steps => [], :flow_id => @name}.merge(data)
+          else
+            metadata_changes = MetadataDSL.new(@name, &bl)
+            @steps << {:type => :graph_metadata, :steps => metadata_changes.to_a, :flow_id => @name}.merge(data)
+          end
+        end
+
+        def to_hash
+          {
+            :name => @name,
+            :steps => @steps
+          }
+        end
+
+        def to_a
+          @steps
+        end
+
+      end
+      
+
+      class MetadataDSL
+
+        def initialize(flow_name, &bl)
+          @name = flow_name
+          @steps = []
+          instance_eval(&bl)
+        end
+
+        def add(data, &bl)
+          fail "Params need to be hash. This means for example remove(:name => \"value\"). " unless data.is_a? Hash
+          fail "Remove field should have at least \"name\" defined. You provided \"#{data}\"." unless data.has_key?(:name)
+          @steps << {:type => :field_add}.merge(data)
+        end
+
+        def remove(data, &bl)
+          fail "Params need to be hash. This means for example remove(:name => \"value\"). " unless data.is_a? Hash
+          fail "Remove field should have at least \"name\" defined. You provided \"#{data}\"." unless data.has_key?(:name)
+          @steps << {:type => :field_remove}.merge(data)
+        end
+
+        def to_hash
+          {
+            :name => @name,
+            :steps => @steps
+          }
+        end
+
+        def to_a
+          @steps
+        end
+
+      end
+
+    end
+  end
+end