gd_bam 0.0.15 → 0.1.0

data/lib/cloud_connect/dsl/cc.rb

@@ -0,0 +1,42 @@
+module GoodData
+  module CloudConnect
+    module Core
+
+      include GoodData::CloudConnect
+
+      def self.build_node2(builder, node)
+        if node[:type] == Nodes::EDGE
+          builder.tag!("Edge", node)
+        elsif node[:type] == Nodes::SF_CONNECTION
+          builder.tag!("Connection", node)
+        elsif node[:type] == Nodes::FILE_LIST
+          builder.tag!("Node", node.remove(:output_mapping)) do |xml|
+            xml.attr({:name => "outputMapping"}) do |attr|
+              transformation = node[:output_mapping]
+              attr.cdata! transformation
+            end
+          end
+        elsif node[:type] == Nodes::REFORMAT || node[:type] == Nodes::EXT_HASH_JOIN
+          builder.tag!("Node", node.remove(:transformation)) do |xml|
+            xml.attr({:name => "transform"}) do |attr|
+              transformation = node[:transformation]
+              attr.cdata! transformation
+            end
+          end
+        elsif node[:type] == Nodes::DATA_GENERATOR
+          builder.tag!("Node", node.remove(:transformation)) do |xml|
+            xml.attr({:name => "generate"}) do |attr|
+              transformation = node[:generate]
+              attr.cdata! transformation
+            end
+          end
+        elsif node[:type] == Nodes::PERSISTENT_LOOKUP || node[:type] == Nodes::GD_LOOKUP
+          builder.tag!("LookupTable", node)
+        else
+          builder.tag!("Node", node)
+        end
+      end
+
+    end
+  end
+end

data/lib/cloud_connect/dsl/es_helpers.rb

@@ -0,0 +1,49 @@
+module GoodData
+  module CloudConnect
+    module Helpers
+
+      def self.create_es_write_json(tap)
+        fail "Only tap should be used here. You provided \"#{tap}\"" unless tap[:type] == :tap
+        {
+          :entityName => tap[:id] || tap[:object],
+          :fieldsMapping => tap[:fields].reduce({}) do |memo, f|
+
+            if f.has_key?(:acts_as)
+
+              f[:acts_as].each do |a|
+                type = case a
+                when "Id"
+                  "recordid"
+                when "Timestamp"
+                  "timestamp"
+                else
+                  f[:type] || "attribute"
+                end
+                memo[a] = {
+                  :name => a,
+                  :type => type
+                }
+              end
+            else
+              type = case f[:name]
+              when "Id"
+                "recordid"
+              when "Timestamp"
+                "timestamp"
+              else
+                f[:type] || "attribute"
+              end
+
+              memo[f[:name]] = {
+                :name => f[:name],
+                :type => type
+              }
+            end
+            memo
+          end
+
+        }
+      end
+    end
+  end
+end

data/lib/cloud_connect/dsl/helpers.rb

@@ -0,0 +1,199 @@
+module GoodData
+  module CloudConnect
+    module Helpers
+
+      def self.property(builder, data)
+        builder.Property({
+          :id => data[:id],
+          :name => data[:name] || data[:id],
+          :value => data[:value]
+        })
+      end
+
+      def self.property_file(builder, data)
+        builder.Property({
+          :id => data[:id],
+          :fileURL => data[:fileURL]
+        })
+      end
+
+      def self.create_trash_meta(builder)
+        builder.Metadata({:id => "trash_metadata"}) do |builder|
+          csv_metadata(builder, {
+            :name   => "trash_metadata",
+            :fields => [{:type => "string", :name => "all"}]
+          })
+        end
+      end
+
+      def self.save_metadata(filename, data)
+        dirname = File.dirname(filename)
+        FileUtils.mkdir_p(dirname) unless File.directory?(dirname)
+ 
+        File.open(filename, "w") do |file|
+          builder = Builder::XmlMarkup.new(:target => file, :indent=>2)
+          builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"  
+          csv_metadata(builder, data)
+        end
+      end
+
+      def self.create_moving_graph(file, options={})
+        source = options[:source]
+        target = options[:target]
+        operation = options[:operation]
+        force = options[:force] || false
+
+        File.open(file, "w") do |file|
+          builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+          builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+          builder.Graph({
+            :name => "File Copy"
+          }) do
+            builder.Global do
+              builder.Metadata({:id   => "list_metadata"}) do |builder|
+                Helpers::csv_metadata(builder, {
+                  :name => "list_metadata",
+                  :fields => [{:name=>"filePath", :type=>"string"}]
+                })
+              end
+              Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+            end
+            builder.Phase(:number => 0) do
+
+              transformation_source = "function integer transform() {\n" + ([["filePath", "filePath"]].map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
+              Core::build_node2(builder, Nodes.file_list2(:baseURL => target, :id => "file_list", :output_mapping => transformation_source))
+
+              Core::build_node2(builder, Nodes.file_delete2(:baseURL => "${filePath}", :id => "file_delete"))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "file_delete:0", :fromNode => "file_list:0", :metadata => "list_metadata"}))
+            end  
+            builder.Phase(:number => 1) do
+              Core::build_node2(builder, Nodes.file_copy2({:sourcePath => source, :targetPath => target, :operation => operation, :id => "file_copy"}))
+            end
+          end
+        end
+      end
+
+      def self.csv_metadata(builder, description)
+        sf_description = description.merge({
+          :fieldDelimiter => ",",
+          :recordDelimiter => "\\n",
+          :type => "delimited",
+        })
+        metadata(builder, sf_description)
+      end
+
+      def self.metadata(builder, description)
+        builder.Record({
+            :fieldDelimiter => description[:fieldDelimiter],
+            :name => description[:name],
+            :recordDelimiter => description[:recordDelimiter],
+            :type => description[:type]
+          }) do |record|
+          description[:fields].each do |field|
+            builder.Field :name => field[:name], :type => field[:type] || "string", :nullable => "true"
+          end
+        end
+      end
+
+      def self.create_lookup_meta(builder)
+        builder.Metadata({:id   => "lookup_metadata"}) do |builder|
+          csv_metadata(builder, {
+            :name   => "lookup_metadata",
+            :fields => [{:type => "string", :name => "key"}, {:type => "string", :name => "value"}]
+          })
+        end
+      end
+
+      def self.create_file_list_meta(builder)
+        builder.Metadata({:id   => "file_list"}) do |builder|
+          csv_metadata(builder, {
+            :name   => "file_list",
+            :fields => [
+              {:name => "filePath", :type => "string"},
+              {:name => "fileName", :type => "string"}
+            ]
+          })
+        end
+      end
+
+      def self.create_read_error_meta(builder)
+         builder.Metadata({:id   => "reader_error_metadata"}) do |builder|
+           csv_metadata(builder, {
+             :name   => "reader_error_metadata",
+             :fields => [
+               {:name => "line_number", :type => "integer"},
+               {:name => "field_number", :type => "integer"},
+               {:name => "record", :type => "string"},
+               {:name => "message", :type => "string"},
+               {:name => "file", :type => "string"}
+             ]
+           })
+         end
+      end
+
+      def self.create_run_graph_failure_metadata(builder)
+        builder.Metadata({:id   => "run_graph_failure_metadata"}) do |builder|
+          csv_metadata(builder, {
+            :name   => "run_graph_failure_metadata",
+            :fields => [
+              {:type => "string", :name => "graph"},
+              {:type => "string", :name => "result"},
+              {:type => "string", :name => "description"},
+              {:type => "string", :name => "message"},
+              {:type => "decimal", :name => "duration"}
+            ]
+          })
+        end
+      end
+
+      def self.create_run_graph(file, options={})
+        subgraphs = options[:subgraphs]
+        flow = options[:flow]
+        File.open(file, "w") do |file|
+          builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+          builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+          builder.Graph({
+            :name => "Run graph"
+          }) do
+            builder.Global do
+              property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+              property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
+              create_trash_meta(builder)
+              create_lookup_meta(builder)
+
+            end
+            phase = 0
+            subgraphs.each do |subgraph|
+              name = GoodData::Bam::Step.step_name(GoodData::Bam::Graph.create(:path => subgraph))
+              # Pathname(subgraph).basename.to_s.chomp(Pathname(subgraph).extname)
+              builder.Phase(:number => phase+1) do
+                id1 = GoodData::CloudConnect::Nodes.get_id
+                id2 = GoodData::CloudConnect::Nodes.get_id
+                ctl = "function integer generate() {$out.0.all = \"FLOW=#{flow}\";return OK;}"
+                Core::build_node2(builder, Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata"}))
+                Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
+              end
+              builder.Phase(:number => phase+2) do
+
+                id1 = GoodData::CloudConnect::Nodes.get_id
+                id2 = GoodData::CloudConnect::Nodes.get_id
+                ctl = "function integer generate() {$out.0.all = \"NAME=#{name}\";return OK;}"
+                Core::build_node2(builder, Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata"}))
+                Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.prm", :outputFieldNames => "false", :append => "true", :quotedStrings => "false"}))
+              end  
+
+              builder.Phase(:number => phase+3) do
+                Core::build_node2(builder, Nodes.run_graph2({:guiName => name, :name => name, :id => flow, :graphName => subgraph}))
+              end
+              phase += 4
+            end
+          end
+        end
+      end
+    
+
+    end
+  end
+end

data/lib/{nodes → cloud_connect/dsl}/nodes.rb

@@ -1,16 +1,21 @@
 module GoodData 
-  module CloverGenerator
+  module CloudConnect
     module Nodes
       
       SF_CONNECTION = "SFDC"  
       EDGE = "EDGE"
       WRITER = "DATA_WRITER"
+      DEDUP = "DEDUP"
       READER = "DATA_READER"
       SF_READER = "SF_READER"
       REFORMAT = "REFORMAT"
+      FILE_BACKUP = "FILE_BACKUP"
       PERSISTENT_LOOKUP = "persistentLookup"
       GD_LOOKUP = "gdLookup"
       GD_ESTORE_WRITER = "GD_ESTORE_WRITER"
+      GD_FILE_BACKUP = "FILE_BACKUP"
+      GD_FILE_COPY = "FILE_COPY_MOVE"
+      FILE_DELETE = "FILE_DELETE"
       GD_ESTORE_READER = "GD_ESTORE_READER"
       ENABLED = "ENABLED"
       DEFAULT_HEIGHT = "77"
@@ -25,12 +30,17 @@ module GoodData
       TRASH = "TRASH"
       LOOKUP_TABLE_READER_WRITER = "LOOKUP_TABLE_READER_WRITER"
       FILE_COPY_MOVE = "FILE_COPY_MOVE"
-      FILE_DELETE = "FILE_DELETE"
       FILE_LIST = "FILE_LIST"
       EXT_SORT = "EXT_SORT"
       EXT_HASH_JOIN = "EXT_HASH_JOIN"
       NORMALIZER = "NORMALIZER"
-      
+
+      MAP_ALL = <<HEREDOC
+function integer transform() {
+    $out.0.* = $in.0.*;
+    return ALL;
+}
+HEREDOC
 
       DEFAULT_NODE_PARAMS = {
         :enabled    => ENABLED,
@@ -38,6 +48,11 @@ module GoodData
         :guiWidth   => DEFAULT_WIDTH
       }
 
+      $__cloud_connect_id = 0
+      def self.get_id()
+        $__cloud_connect_id += 1
+      end
+
       def self.node(data, options={})
         defaults = options[:defaults] || {}
         required = options[:required] || []
@@ -45,7 +60,7 @@ module GoodData
         output_data = defaults.merge(data)
         required.each do |key|
           unless output_data.has_key?(key)
-            fail "#{key} is required but not provided"
+            fail "#{key} is required but not provided. You defined \"#{data}\""
           end
         end
         output_data
@@ -55,7 +70,9 @@ module GoodData
         defaults = DEFAULT_NODE_PARAMS.merge(options[:defaults] || {})
         required = (options[:required] || []).concat([:type, :id])
         allowed = options[:allowed] || []
-        output_data = defaults.merge(data)
+        output_data = defaults.merge({
+          :id => get_id()
+        }).merge(data)
 
         node(output_data, {
           :allowed => [:enabled, :guiName, :id, :type, :guiHeight, :guiWidth, :name].concat(allowed),
@@ -113,6 +130,67 @@ module GoodData
         })
       end
 
+
+
+      def self.file_delete2(data, options={})
+        local_defaults = {
+          :type => FILE_DELETE
+        }
+        defaults = local_defaults.merge(options[:defaults] || {})
+
+        base_node(defaults.merge(data), {
+          :allowed => [:baseURL],
+          :defaults => defaults,
+          :required => [:baseURL]
+        })
+      end
+
+      def self.backup2(data, options={})
+        local_defaults = {
+          :type => GD_FILE_BACKUP,
+          :mode => "ALL_TO_ONE",
+          :appendTimestamp => true,
+          :makeDirs => true
+        }
+        defaults = local_defaults.merge(options[:defaults] || {})
+
+        base_node(defaults.merge(data), {
+          :allowed => [:sourcePath, :makeDirs, :appendTimestamp, :mode, :baseURL],
+          :defaults => defaults,
+          :required => []
+        })
+      end
+      
+      def self.file_backup2(data, options={})
+        local_defaults = {
+          :type => FILE_BACKUP,
+          :makeDirs => true
+        }
+        defaults = local_defaults.merge(options[:defaults] || {})
+
+        base_node(defaults.merge(data), {
+          :allowed => [:sourcePath, :baseURL, :mode, :makeDirs],
+          :defaults => defaults,
+          :required => []
+        })
+      end
+      
+      def self.file_copy2(data, options={})
+        local_defaults = {
+          :type => GD_FILE_COPY,
+          :operation => "COPY",
+          :makeDirs => true
+        }
+        defaults = local_defaults.merge(options[:defaults] || {})
+
+        base_node(defaults.merge(data), {
+          :allowed => [:sourcePath, :targetPath, :makeDirs, :operation, :baseURL],
+          :defaults => defaults,
+          :required => []
+        })
+      end
+      
+
       def self.sfdc_reader2(data, options={})
         local_defaults = {
           :type => SF_READER,
@@ -127,6 +205,20 @@ module GoodData
 
       end
 
+      def self.dedup2(data, options={})
+        local_defaults = {
+          :type => DEDUP,
+        }
+        defaults = local_defaults.merge(options[:defaults] || {})
+
+        base_node(defaults.merge(data), {
+          :allowed => [],
+          :defaults => defaults,
+          :required => [:dedupKey]
+        })
+
+      end
+
       def self.run_graph2(data, options={})
         local_defaults = {
           :type => RUN_GRAPH,
@@ -136,8 +228,7 @@ module GoodData
 
         base_node(defaults.merge(data), {
           :allowed => [:paramsToPass, :type, :name, :clientId, :loginHostname, :name, :username, :password, :passwordEncrypted, :token, :id],
-          :defaults => defaults,
-          :required => [:graphName]
+          :defaults => defaults
         })
       end
 
@@ -247,7 +338,7 @@ module GoodData
         base_node(defaults.merge(data), {
           :allowed => [],
           :defaults => defaults,
-          :required => []
+          :required => [:output_mapping]
         })
       end
 
@@ -312,10 +403,11 @@ module GoodData
         })
       end
 
-
-      def self.file_delete2(data, options={})
+      def self.file_copy2(data, options={})
         local_defaults = {
-          :type           => FILE_DELETE
+          :type           => FILE_COPY_MOVE,
+          :operation      => "COPY",
+          :baseURL        => "${PROJECT}"
         }
         defaults = local_defaults.merge(options[:defaults] || {})
         base_node(defaults.merge(data), {
@@ -325,11 +417,9 @@ module GoodData
         })
       end
 
-      def self.file_copy2(data, options={})
+      def self.gather2(data, options={})
         local_defaults = {
-          :type           => FILE_COPY_MOVE,
-          :operation      => "COPY",
-          :baseURL        => "${PROJECT}"
+          :type           => SIMPLE_GATHER,
         }
         defaults = local_defaults.merge(options[:defaults] || {})
         base_node(defaults.merge(data), {
@@ -338,7 +428,7 @@ module GoodData
           :required => []
         })
       end
-
+      
 
       def self.copy2(data, options={})
         local_defaults = {

data/lib/cloud_connect/dsl/sf_helpers.rb

@@ -0,0 +1,39 @@
+module GoodData
+  module CloudConnect
+    module Helpers
+
+      def self.generate_select(tap)
+        fields = tap[:fields].map do |f| 
+          f.has_key?(:multi_currency) ? "convertCurrency(#{f[:name]})" : f[:name]
+        end
+        condition = tap[:condition].nil?() ? "" : "WHERE #{tap[:condition]}"
+        limit = tap[:limit].nil?() ? "" : "LIMIT #{tap[:limit]}"
+        "SELECT #{fields.join(', ')} FROM #{tap[:object]} #{condition} #{limit}"
+      end
+
+      def self.generate_incremental_select(tap)
+        if tap[:condition].blank?
+          tap[:condition] = "SystemModstamp > ${#{tap[:id]}_START} AND SystemModstamp <= ${#{tap[:id]}_END}"
+        else
+          tap[:condition] += " AND SystemModstamp > ${#{tap[:id]}_START} AND SystemModstamp <= ${#{tap[:id]}_END}"
+        end
+        generate_select(tap)
+      end
+
+      def self.sf_connection(builder, data)
+        builder.Connection({
+          :clientId => "${SFDC_CLIENT_ID}",
+          :id => "SFDC",
+          :loginHostname => "${SFDC_LOGIN_HOSTNAME}",
+          :name => "${SFDC_NAME}",
+          :password => "${SFDC_PASSWORD}",
+          :passwordEncrypted => "true",
+          :token => "${SFDC_TOKEN}",
+          :type => "SFDC",
+          :username => "${SFDC_USERNAME}",
+          :passwordEncrypted => "false"
+        })
+      end
+    end
+  end
+end

data/lib/cloud_connect/dsl/structure_helpers.rb

@@ -0,0 +1,94 @@
+module GoodData
+  module CloudConnect
+    module Helpers
+
+      include GoodData::CloudConnect
+
+      def self.loop_over_file(file, options={})
+        file_to_loop = options[:file_to_loop]
+        token = options[:token]
+        graph_to_run = options[:graph_to_run]
+
+        fail "token not defined" if token.blank?
+        fail "file to loop if not defined" if file_to_loop.blank?
+        fail "graph_to_run not defined" if graph_to_run.blank?
+
+
+        File.open(file, "w") do |f|
+
+          builder = Builder::XmlMarkup.new(:target=>f, :indent=>2)
+          builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+          builder.Graph({
+            :name => "Contract Checker: File list reader"
+          }) do
+            builder.Global do
+              Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+              Helpers::property_file(builder, {:id => "skiplines_params", :fileURL => "#{token}_counter.prm"})
+
+              Helpers::create_trash_meta(builder)
+              Helpers::create_run_graph_failure_metadata(builder)
+              Helpers::create_file_list_meta(builder)
+
+            end
+
+          reformat_func = <<HEREDOC
+function integer transform() {
+    $out.0.all = "FILE=" + $in.0.filePath;
+    return ALL;
+}
+HEREDOC
+
+            builder.Phase(:number => 0) do
+              Core::build_node2(builder, Nodes.reader2({
+                :id => "#{token}_reader",
+                :guiName => "#{token}_reader",
+                :name => "#{token}_reader",
+                :dataPolicy => "Strict",
+                :skipRows => "${#{token}_SKIP_LINES}",
+                :numRecords => 1,
+                :fileURL => file_to_loop
+              }))
+              Core::build_node2(builder, Nodes.reformat2({:name => "#{token} Reformat", :id => "#{token}_reformat", :transformation => reformat_func }))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{token}_reformat:0", :fromNode => "#{token}_reader:0", :metadata => "file_list" }))
+              Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{token}_writer", :fileURL => "#{token}_item.prm" , :outputFieldNames => "false", :quotedStrings => "false"}))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{token}_writer:0", :fromNode => "#{token}_reformat:0", :metadata => "trash_metadata" }))
+            end
+            builder.Phase(:number => 1) do 
+
+          reformat_func = <<HEREDOC
+function integer transform() {
+    $out.0.all = "#{graph_to_run}";
+    return ALL;
+}
+HEREDOC
+
+          fail_func = <<HEREDOC
+function integer transform() {
+  raiseError("Graph #{graph_to_run} failed");
+}
+HEREDOC
+              Core::build_node2(builder, Nodes.reformat2({:name => "#{token} Reformat_graph", :id => "#{token}_reformat_graph", :transformation => reformat_func }))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{token}_reformat_graph:0", :fromNode => "#{token}_reformat:1", :metadata => "trash_metadata" }))
+              Core::build_node2(builder, Nodes.run_graph2({:guiName => "#{token}_validate", :name => "#{token}_validate", :id => "#{token}_validate" }))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{token}_validate:0", :fromNode => "#{token}_reformat_graph:0", :metadata => "trash_metadata" }))
+              Core::build_node2(builder, Nodes.reformat2({:name => "#{token} Reformat_fail", :id => "#{token}_reformat_fail", :transformation => fail_func }))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{token}_reformat_fail:0", :fromNode => "#{token}_validate:1", :metadata => "run_graph_failure_metadata" }))
+              Core::build_node2(builder, Nodes.trash2({:name => "#{token}_trash", :id => "#{token}_trash", :debugPrint => true}))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{token}_trash:0", :fromNode => "#{token}_reformat_fail:0", :metadata => "run_graph_failure_metadata"}))
+
+            end
+
+            builder.Phase(:number => 2) do
+
+
+              ctl = "function integer generate() {$out.0.all = \"#{token}_SKIP_LINES=\" + toString(${#{token}_SKIP_LINES} + 1);return OK;}"
+              Core::build_node2(builder, Nodes.data_generator2({:name => "#{token}_generator", :id => "#{token}_generator", :generate => ctl}))
+              Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{token}_csv_writer", :fileURL => "#{token}_counter.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{token}_csv_writer:0", :fromNode => "#{token}_generator:0", :metadata => "trash_metadata" }))
+            end
+          end
+        end
+      end
+    end
+  end
+end