gd_bam 0.0.15 → 0.1.0

data/lib/generators/etl.rb

@@ -0,0 +1,261 @@
+module GoodData
+  module Bam
+    module Generators
+      module Etl
+
+        include GoodData::Bam
+        include GoodData::CloudConnect
+
+
+        def self.create_es_readmap(metadata, tap)
+          e = Es::Entity.new("x", {
+            :file => "none",
+            :fields => metadata[:fields].map do |f|
+              name = f[:name]
+              if name == "Id"
+                Es::Field.new('Id', 'recordid')
+              elsif name == "Snapshot"
+                Es::SnapshotField.new('Snapshot', 'snapshot')
+              else
+                Es::Field.new(name, 'attribute')
+              end
+            end,
+            :timeframe => Es::Timeframe::parse("latest")
+          })
+
+
+          stuff = {
+            :entityName => tap[:id],
+            :fieldsMapping => tap[:fields].inject({}) do |memo, field|
+              name = field[:name]
+              memo[name] = name
+              memo
+            end,
+            :eventStoreFieldToTypeMapping => tap[:fields].inject({}) do |memo, field|
+              name = field[:name]
+              if name == "Id"
+                memo[name] = "recordid"
+              elsif name == "Snapshot"
+                memo[name] = "snapshot"
+              else
+                memo[name] = "attribute"
+              end
+
+              memo
+            end,
+            :outputMetadataName => "es_metadata"
+          }
+
+          read_map = {
+            :columns => e.to_extract_fragment('pid')["readTask"]["readMap"].first[:columns],
+            :populates => e.to_extract_fragment('pid')["readTask"]["readMap"].first[:populates]
+          }
+          if tap.has_key?(:time_intervals)
+            [stuff, read_map, tap[:time_intervals]]
+          else
+            [stuff, read_map, nil]
+          end
+          
+        end
+
+        def self.create_file_uploading_graph(file, sink, input_name, state={})
+          File.open(file, "w") do |file|
+            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+            builder.Graph({
+              :name => "GoodData Data loader"
+            }) do
+              builder.Global do
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+                builder.Metadata({:id => "load_metadata"}) do |builder| 
+                  Helpers::csv_metadata(builder, state[:metadata][input_name])
+                end
+              end
+              builder.Phase(:number => 0) do
+                gd_dataset = sink[:gd_name] || sink[:id]
+                to_svinstvo = GoodData::CloverGenerator::build_gd_dataset_loader_json(sink)
+                Core::build_node2(builder, Nodes.reader2({:name => "CSV Loader", :id => "loader", :fileURL => "${PROJECT}/data/#{input_name}.csv"}))
+                Core::build_node2(builder, Nodes.writer2({:name => "GD Loader", :id => "writer", :fileURL => sink[:target]}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "writer:0", :fromNode => "loader:0", :metadata => "load_metadata"}))
+              end
+            end
+          end
+        end
+
+        def self.create_gd_uploading_graph(file, sink, input_name, state={})
+          File.open(file, "w") do |file|
+            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+            builder.Graph({
+              :name => "GoodData Data loader"
+            }) do
+              builder.Global do
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+                builder.Metadata({:id => "load_metadata"}) do |builder| 
+                  Helpers::csv_metadata(builder, state[:metadata][input_name])
+                end
+              end
+              builder.Phase(:number => 0) do
+                gd_dataset = sink[:gd_name] || sink[:id]
+                to_svinstvo = GoodData::CloverGenerator::build_gd_dataset_loader_json(sink)
+                Core::build_node2(builder, Nodes.reader2({:name => "CSV Loader", :id => "loader", :fileURL => "${PROJECT}/data/#{input_name}.csv"}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "gd_load:0", :fromNode => "loader:0", :metadata => "load_metadata"}))
+                Core::build_node2(builder, Nodes.gd_loader2({:name => "GD Loader", :id => "gd_load", :dataset => "dataset.#{gd_dataset}", :datasetFieldMappings => to_svinstvo.to_json}))
+              end
+            end
+          end
+        end
+
+        def self.create_file_downloading_graph(file, tap, options={})
+          s3_backup = options[:params][:s3_backup]
+
+          File.open(file, "w") do |file|
+            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+            builder.Graph({
+              :name => "File Downloader"
+            }) do
+              builder.Global do
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+
+                builder.Metadata({:id   => "file_metadata"}) do |builder|
+                  Helpers::csv_metadata(builder, Metadata.get_source_metadata(tap))
+                end
+                builder.Metadata({:id   => "clover_metadata"}) do |builder|
+                  Helpers::csv_metadata(builder, Metadata.get_target_metadata(tap))
+                end
+              end
+
+              builder.Phase(:number => 0) do
+                Core::build_node2(builder, Nodes.reader2({:name => "reader", :id => "reader", :fileURL => tap[:source] }))
+
+                Core::build_node2(builder, Nodes.copy2({:name => "copy", :id => "copy"}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "reader:0", :metadata => "file_metadata"}))
+
+                transformation_source = "function integer transform() {\n" + (Tap.source_target_mappging(tap).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
+                Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "reformat", :transformation => transformation_source}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "reformat:0", :fromNode => "copy:0", :metadata => "file_metadata"}))
+
+                Core::build_node2(builder, Nodes.writer2({:name => "CSV Writer", :id => "csv", :fileURL => "data/#{tap[:id]}.csv", :outputFieldNames => "true"}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "csv:0", :fromNode => "reformat:0", :metadata => "clover_metadata"}))
+
+                if s3_backup then Core::build_node2(builder, Nodes.writer2({:name => "s3 Writer", :id => "s3", :fileURL => "bucket", :outputFieldNames => true})) end
+                if s3_backup then Core::build_node2(builder, Nodes.edge2({:toNode => "s3:0", :fromNode => "copy:1", :metadata => "file_metadata"})) end
+              end
+            end
+          end
+        end
+
+        def self.create_sf_downloading_graph(file, tap, options={})
+          s3_backup = options[:params][:s3_backup]
+
+          File.open(file, "w") do |file|
+            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+            builder.Graph({
+              :name => "Goodsales Salesforce Downloader"
+            }) do
+              builder.Global do
+
+                builder.Metadata({:id   => "clover_metadata"}) do |builder|
+                  Helpers::csv_metadata(builder, Metadata.get_target_metadata(tap))
+                end
+
+                builder.Metadata({:id   => "sf_metadata"}) do |builder|
+                  Helpers::csv_metadata(builder, Metadata.get_source_metadata(tap))
+                end
+
+                Helpers::sf_connection(builder, {})
+                Helpers::property(builder, {:id => "SFDC_CLIENT_ID", :value => "gooddata/gooddata/"})
+                Helpers::property(builder, {:id => "SFDC_LOGIN_HOSTNAME", :value => options[:params][:sf_server] || "login.salesforce.com"})
+                Helpers::property(builder, {:id => "SFDC_NAME", :value => "Salesforce connection"})
+                Helpers::property(builder, {:id => "SFDC_PASSWORD", :value => options[:params][:sf_password]})
+                Helpers::property(builder, {:id => "SFDC_TOKEN", :value => options[:params][:sf_token]})
+                Helpers::property(builder, {:id => "SFDC_USERNAME", :value => options[:params][:sf_login]})
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+              end
+
+              builder.Phase(:number => 0) do
+                fields = tap[:fields]
+                mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
+                add = fields.map do |f|
+                  "{\"xmlFieldMapping\":{\"name\":\"#{f[:name]}\",\"label\":\"#{f[:label]}\",\"xmlPath\":\"#{f[:name]}\",\"metadataField\":\"#{f[:name]}\"}}"
+                end
+
+                stuff = mapping + add.join(",") + "]}}"
+                mandatory_fields = fields.reject {|f| f[:is_mandatory] == false }.map {|f| f[:name] + ";"}.join
+
+                Core::build_node2(builder, Nodes.sfdc_reader2({:name => "SF Writer", :id => "sf", :soql => Helpers::generate_select(tap), :sfdcConnection => "SFDC", :fieldsMapping => stuff, :mandatoryFields => mandatory_fields}))
+
+                Core::build_node2(builder, Nodes.copy2({:name => "copy", :id => "copy"}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "sf:0", :metadata => "sf_metadata"}))
+
+                transformation_source = "function integer transform() {\n" + (Tap.source_target_mappging(tap).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
+                Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "reformat", :transformation => transformation_source}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "reformat:0", :fromNode => "copy:0", :metadata => "sf_metadata"}))
+
+                Core::build_node2(builder, Nodes.writer2({:name => "CSV Writer", :id => "csv", :fileURL => "data/#{tap[:id]}.csv", :outputFieldNames => "true"}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "csv:0", :fromNode => "reformat:0", :metadata => "clover_metadata"}))
+                if s3_backup
+                  Core::build_node2(builder, Nodes.writer2({:name => "s3 Writer", :id => "s3", :fileURL => "bucket", :outputFieldNames => true}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "s3:0", :fromNode => "copy:1", :metadata => "sf_metadata"}))
+                end
+              end
+            end
+          end
+        end
+
+        def self.create_es_downloading_graph(file, tap, options={})
+          s3_backup = options[:params][:s3_backup]
+
+          metadata_from_es = Metadata.get_source_metadata(tap)
+          output_metadata = Metadata.get_target_metadata(tap)
+          id = tap[:id]
+
+          File.open(file, "w") do |file|
+            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+            builder.Graph({
+              :name => "GoodData EventStore Downloader"
+            }) do
+              builder.Global do
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+
+                builder.Metadata({:id   => "es_metadata"}) do |builder|
+                  Helpers::csv_metadata(builder, metadata_from_es)
+                end
+
+                builder.Metadata({:id   => "output_metadata"}) do |builder|
+                  Helpers::csv_metadata(builder, output_metadata)
+                end
+              end
+
+              builder.Phase(:number => 0) do
+                stuff, read_map, time_frames = create_es_readmap(metadata_from_es, tap)
+                
+                if time_frames.blank?
+                  Core::build_node2(builder, Nodes.es_reader2({:name => "ES Reader", :id => "es", :entityFieldsMapping => stuff.to_json , :readMap => read_map.to_json}))
+                else
+                  Core::build_node2(builder, Nodes.es_reader2({:name => "ES Reader", :id => "es", :entityFieldsMapping => stuff.to_json , :readMap => read_map.to_json, :timeIntervals => time_frames.to_json}))
+                end
+                
+                transformation_source = if Metadata::has_field?(metadata_from_es, "Snapshot")
+                  "function integer transform() {\n" + (Tap.source_target_mappging(tap).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\n$out.0.Snapshot = date2str(dateAdd(str2date(\"1900-01-01\", \"yyyy-MM-dd\"),  str2long($in.0.Snapshot), day), \"yyyy-MM-dd\");\nreturn OK;\n}"
+                else
+                  "function integer transform() {\n" + (Tap.source_target_mappging(tap).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
+                end
+                Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "reformat", :transformation => transformation_source}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "reformat:0", :fromNode => "es:0", :metadata => "es_metadata"}))
+
+                Core::build_node2(builder, Nodes.copy2({:name => "Copy", :id => "copy"}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "reformat:0", :metadata => "output_metadata"}))
+                Core::build_node2(builder, Nodes.writer2({:name => "CSV Writer", :id => "csv", :fileURL => "${PROJECT}/data/#{tap[:id]}.csv", :outputFieldNames => "true", :makeDirs => "true"}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "csv:0", :fromNode => "copy:0", :metadata => "output_metadata"}))
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end