gd_bam 0.1.35 → 0.1.36

data/bin/bam

@@ -21,6 +21,17 @@ default_value false
 arg_name 'logger'
 switch [:l,:logger]
 
+desc 'Platform server'
+default_value nil
+arg_name 'server'
+flag [:s,:server]
+
+desc 'Platform uploads server'
+default_value nil
+arg_name 'web dav upload server'
+flag [:w,:webdav_server]
+
+
 desc 'Generates clover project based on information in current directory. The default ouptut is the directory ./clover-project'
 # arg_name 'Describe arguments to new here'
 command :generate do |c|
@@ -99,7 +110,7 @@ command :run_validator do |c|
     end
     fail "You need to specify process" if process.blank?
     
-    GoodData::Bam::Commands::connect_to_gd()
+    GoodData::Bam::Commands::connect_to_gd(global_options)
     GoodData::Bam::Commands::run_validator(process, files, PARAMS)
   end
 end
@@ -240,6 +251,62 @@ command :procs do |c|
 end
 
 
+desc 'Replicate and join S3 backup locally. Use ONLY if you knwo what it does.'
+command :clone_backup_locally do |c|
+
+  c.desc 'taps'
+  c.arg_name 'taps'
+  c.flag :taps
+  
+
+  c.action do |global_options,options,args|
+    only = options[:only]
+    params = PARAMS.merge({
+      :project_name => "backup-restore-#{PARAMS[:project_name]}",
+      :graph_repos => [
+        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
+        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)],
+      :only => only
+    })
+    GoodData::Bam::Commands::clone_backup(".", params)
+  end
+
+end
+desc 'Restore data from S3 backup'
+command :restore_backup do |c|
+
+  c.desc 'ES name'
+  c.arg_name 'es'
+  c.flag :es
+
+  c.desc 'specify tap to process only that one'
+  c.arg_name 'only'
+  c.flag :only
+  
+
+  c.action do |global_options,options,args|
+    new_es = options[:es]
+    only = options[:only]
+    params = PARAMS.merge({
+      :project_name => "backup-restore-#{PARAMS[:project_name]}",
+      :graph_repos => [
+        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
+        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)],
+      :new_es => new_es,
+      :only => only
+    })
+
+    fail "You do not have backup information specified. I do not know where to look for the backup." unless PARAMS[:s3_backup]
+    fail "You did not specify an ES." if new_es.blank?
+    fail "ES cannot be the same as you currently have in params.json. The point is to use a new one. If anything goes wrong you can always go back." if new_es == (params[:additional_params] && params[:additional_params][:GDC_EVENTSTORE])
+
+    # puts "Let's reload backup"
+    GoodData::Bam::Commands::generate_backup_restoration(".", params)
+  end
+
+end
+
+
 desc 'Creates project'
 command :project do |c|
 
@@ -253,20 +320,22 @@ command :project do |c|
 
   c.action do |global_options,options,args|
     fail "You need to specify token to create a project" if options[:token].nil?
-    GoodData::Bam::Commands::connect_to_gd()
-    pid = case options[:blueprint]
+    GoodData::Bam::Commands::connect_to_gd(global_options)
+    creation_info = case options[:blueprint]
       when "goodsales"
-        "nt935rwzls50zfqwy6dh62tabu8h0ocy"
+        { :template => "/projectTemplates/GoodSalesAnalytics/2" }
       when "box"
-        "n1zoyqvahupyxfxv35nzo0ernax69odm"
+        { :pid => "n1zoyqvahupyxfxv35nzo0ernax69odm" }
     end
 
     params = PARAMS.merge({:token => options[:token]})
 
-    new_project = if pid
-      GoodData::Bam::Commands.clone_project(pid, params)
-    else
+    new_project = if creation_info && creation_info.has_key?(:template)
+      params = PARAMS.merge({:token => options[:token], :template => creation_info[:template]})
+      
       GoodData::Bam::Commands.create_project(params)
+    else
+      GoodData::Bam::Commands.clone_project(pid, params)
     end
 
     puts "Your project pid is #{new_project}"
@@ -360,7 +429,7 @@ command :deploy do |c|
     fail "You have to specify directory to deploy as an argument" if dir.nil?
     fail "Specified directory does not exist" unless File.exist?(dir)
     
-    GoodData::Bam::Commands::connect_to_gd()
+    GoodData::Bam::Commands::connect_to_gd(global_options)
     params = PARAMS.merge(global_options.merge(options))
     response = GoodData::Bam::Commands::deploy(dir, params)
   end
@@ -380,7 +449,7 @@ command :run do |c|
     fail "Specified directory does not exist" unless File.exist?(dir)
 
     params = PARAMS.merge(global_options.merge(options))
-    GoodData::Bam::Commands::connect_to_gd()
+    GoodData::Bam::Commands::connect_to_gd(global_options)
     GoodData::Bam::Commands::run(dir, params)
   end
 end

data/lib/bam/version.rb

@@ -1,3 +1,3 @@
 module Bam
-  VERSION = '0.1.35'
+  VERSION = '0.1.36'
 end

data/lib/cloud_connect/dsl/sf_helpers.rb

@@ -45,7 +45,11 @@ module GoodData
           :token => "${SFDC_TOKEN}",
           :type => "SFDC",
           :username => "${SFDC_USERNAME}",
-          :passwordEncrypted => "false"
+          :passwordEncrypted => "false",
+          :consumerKey => "${SFDC_CONSUMER_KEY}",
+          :consumerSecret => "${SFDC_CONSUMER_SECRET}",
+          :credentialsType => "${SFDC_CREDENTIALS_TYPE}",
+          :refreshToken => "${SFDC_REFRESH_TOKEN}"
         })
       end
     end

data/lib/cloud_connect/dsl/structure_helpers.rb

@@ -4,6 +4,10 @@ module GoodData
 
       include GoodData::CloudConnect
 
+      # Structural pattern
+      # loop_over_file will loop over every line provided in the file given as parameter
+      # it will run the graph given as options[:graph_to_run]
+      # it will make all arrangments that the underlying graph will have access to only one line thus making a loop
       def self.loop_over_file(file, options={})
         file_to_loop = options[:file_to_loop]
         token = options[:token]

data/lib/commands/commands.rb

@@ -103,6 +103,98 @@ module GoodData
         GoodData::Bam::Generators::Downloaders.generate_history_downloaders(home + DOWNLOADERS_HOME + "graphs", project, params)
       end
 
+      def self.generate_backup_restoration(home, params)
+        home = Pathname(home)
+        new_params = params.clone
+        only = params[:only]
+        new_params[:additional_params][:GDC_EVENTSTORE] = params[:new_es]
+        clobber_downloaders_project(home)
+        project = GoodData::Bam::Project.build_project(home, new_params)
+        
+        require 'aws-sdk'
+        AWS.config(
+          :access_key_id     => params[:additional_params][:S3_ACCESS_KEY_ID],
+          :secret_access_key => params[:additional_params][:S3_SECRET_ACCESS_KEY]
+        )
+
+        s3_interface_from = AWS::S3.new()
+        bucket_from = s3_interface_from.buckets[params[:additional_params][:S3_BUCKETNAME]]
+
+        files_in_from_bucket = bucket_from.objects.map {|o| o.key}
+
+        incremental_taps = Taps.get_incremental(project[:taps])
+
+        setup_downloaders_project(home, new_params)
+        GoodData::Bam::Generators::Downloaders.generate_backup_restoration_graph(home + DOWNLOADERS_HOME + "graphs", project, new_params)
+
+      end
+
+      def self.clone_backup(home, params)
+        home = Pathname(home)
+        new_params = params.clone
+        only = params[:only]
+        new_params[:additional_params][:GDC_EVENTSTORE] = params[:new_es]
+        clobber_downloaders_project(home)
+        project = GoodData::Bam::Project.build_project(home, new_params)
+        
+        require 'aws-sdk'
+        AWS.config(
+          :access_key_id     => params[:additional_params][:S3_ACCESS_KEY_ID],
+          :secret_access_key => params[:additional_params][:S3_SECRET_ACCESS_KEY]
+        )
+
+        s3_interface_from = AWS::S3.new()
+        bucket_from = s3_interface_from.buckets[params[:additional_params][:S3_BUCKETNAME]]
+
+        files_in_from_bucket = bucket_from.objects.map {|o| o.key}
+
+        binding.pry
+
+        incremental_taps = Taps.get_incremental(project[:taps])
+
+        taps_to_process = if only.blank?
+          incremental_taps
+        else
+          # binding.pry
+           temp = Taps.get_incremental([Project::find_tap_by_id(project, only)].reject {|x| x.nil?})
+           fail "There is no such tap \"#{only}\"" if temp.empty?
+           temp
+        end
+        
+        taps_to_process.each do |tap|
+          what = tap[:id]
+          Dir.mktmpdir do |dir|
+            dir = Pathname(dir)
+            files_in_from_bucket.grep(/#{params[:project_pid]}\/#{what}\/#{what}/).each do |file_name|
+              puts "Downloading #{file_name}"
+              o = bucket_from.objects[file_name]
+              local_File_name = file_name.split("\/").last
+              # puts local_File_name
+              File.open(dir + local_File_name,"w") do |local_file|
+                local_file.write(o.read)
+              end
+            end
+            
+            # FileUtils::cd(dir) do
+            full_file_name = "full_" + what + ".csv"
+            # TODO
+              system "\"line to be skipped\n\">> #{dir + full_file_name}"
+              
+              system "for i in #{dir + what}*
+                do
+                  if test -f \"$i\"
+                  then
+                     echo \"Doing somthing to $i\"
+                     tail +2 \"$i\" >> #{dir + full_file_name}
+                  fi
+              done"
+              FileUtils::cp(dir + full_file_name, full_file_name)
+              # binding.pry
+            # end
+          end
+        end
+      end
+
       def self.generate(home, params)
         # fail "The flow you specified was not found" if flows.empty? && !only_flow.nil?
         clobber_etl_project(home)

data/lib/commands/gd_commands.rb

@@ -3,8 +3,9 @@ module GoodData
     module Commands
 
       def self.connect_to_gd(options={})
-          GoodData.connect(PARAMS[:gd_login], PARAMS[:gd_pass])
-          GoodData.project = PARAMS[:project_pid] if !PARAMS[:project_pid].nil? && !PARAMS[:project_pid].empty?
+        server = options[:server]
+        GoodData.connect(PARAMS[:gd_login], PARAMS[:gd_pass], server, options)
+        GoodData.project = PARAMS[:project_pid] if !PARAMS[:project_pid].nil? && !PARAMS[:project_pid].empty?
       end
 
       def self.set_up_debug(project, flow, graph)
@@ -73,15 +74,18 @@ module GoodData
           :project => {
             :content => {
               :guidedNavigation => 1,
-              :driver => "Pg",
+              :driver => "Pg", 
               :authorizationToken => options[:token]
             },
             :meta => {
               :title => project_name,
-              :summary => "Testing Project"
+              :summary => "Testing Project",
+              :projectTemplate => options[:template]
             }
           }
         }
+
+        binding.pry
         result = GoodData.post("/gdc/projects/", pr)
         uri = result["uri"]
         while(GoodData.get(uri)["project"]["content"]["state"] == "LOADING")

data/lib/generators/downloaders.rb

@@ -139,68 +139,150 @@ HEREDOC
               end
 
               file_taps = Taps.get_file(taps)
+              generate_file_lists(file_taps, {
+                phase => phase,
+                :builder => builder,
+                :reformat_func => :webdav
+              })
+#               file_taps.each do |tap|
+#                 source = tap[:source]
+#                 id =  tap[:id]
+# 
+# 
+#                 reformat_func = <<HEREDOC
+# 
+# function integer transform() {
+#     $out.0.filePath = replace($in.0.filePath, "${GDC_WEBDAV_HOST}", replace(replace(\"${GD_LOGIN}\",\"@\",\"%40\"),\"\\\\+\",\"%2B\") + ":${GD_PASSWORD}@${GDC_WEBDAV_HOST}");
+#     $out.0.fileName = $in.0.fileName;
+#     return ALL;
+# }
+# HEREDOC
+# 
+#                 builder.Phase(:number => phase += 1) do
+#                   Core::build_node2(builder, Nodes.file_list2({:id => "#{id}_file_list", :name => "#{id}_file_list", :dataPolicy => "Strict", :baseURL => "#{tap[:source]}", :output_mapping => Nodes::MAP_ALL}))
+#                   Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat", :id => "#{id}_reformat", :transformation => reformat_func}))
+#                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat:0", :fromNode => "#{id}_file_list:0", :metadata => "file_list"}))
+#                   Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_writer", :fileURL => "data/#{id}_files_to_read.csv"}))
+#                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_writer:0", :fromNode => "#{id}_reformat:0", :metadata => "file_list"}))
+#                 end
+# 
+#                 builder.Phase(:number => phase += 1) do
+#                   ctl = "function integer generate() {$out.0.all = \"#{id}_SKIP_LINES=0\";return OK;}"
+#                   Core::build_node2(builder, Nodes.data_generator2({:name => "#{id}_generator", :id => "#{id}_generator", :generate => ctl}))
+#                   Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_csv_writer", :fileURL => "#{id}_counter.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
+#                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_csv_writer:0", :fromNode => "#{id}_generator:0", :metadata => "trash_metadata"}))
+#                 end
+# 
+#                subgraph_reformat_func = <<HEREDOC
+# 
+# function integer transform() {
+#     $out.0.all = "graphs/#{id}_loop.grf";  
+#     return ALL;
+# }
+# HEREDOC
+# 
+#             fail_reformat = <<HEREDOC
+# function integer transform() {
+#   raiseError("Loop failed");
+# }
+# HEREDOC
+# 
+# 
+#                 builder.Phase(:number => phase += 1) do
+#                   Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat graph", :id => "#{id}_reformat_graph", :transformation => subgraph_reformat_func}))
+#                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_graph:0", :fromNode => "#{id}_reformat:1", :metadata => "file_list"}))
+#                   Core::build_node2(builder, Nodes.run_graph2({
+#                     :guiName => id,
+#                     :name => id,
+#                     :id => "#{id}_run_graph"
+#                   }))
+#                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_run_graph:0", :fromNode => "#{id}_reformat_graph:0", :metadata => "trash_metadata"}))
+#                   Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat fail", :id => "#{id}_reformat_fail", :transformation => fail_reformat}))
+#                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_fail:0", :fromNode => "#{id}_run_graph:1", :metadata => "run_graph_failure_metadata"}))
+#                   Core::build_node2(builder, Nodes.trash2({:name => "#{id}_trash", :id => "#{id}_trash", :debugPrint => true}))
+#                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_trash:0", :fromNode => "#{id}_reformat_fail:0", :metadata => "run_graph_failure_metadata"}))
+#                 end
+#               end
 
-              file_taps.each do |tap|
-                source = tap[:source]
-                id =  tap[:id]
 
+            end
+          end
+        end
 
+        def self.generate_file_lists(taps, options={})
+          phase = options[:phase] || 0
+          builder = options[:builder] || fail("I need a builder")
+          reformat_func = nil
+          case options[:reformat_func]
+          when :webdav
             reformat_func = <<HEREDOC
 
 function integer transform() {
-    $out.0.filePath = replace($in.0.filePath, "${GDC_WEBDAV_HOST}", replace(replace(\"${GD_LOGIN}\",\"@\",\"%40\"),\"\\\\+\",\"%2B\") + ":${GD_PASSWORD}@${GDC_WEBDAV_HOST}");
-    $out.0.fileName = $in.0.fileName;
-    return ALL;
+$out.0.filePath = replace($in.0.filePath, "${GDC_WEBDAV_HOST}", replace(replace(\"${GD_LOGIN}\",\"@\",\"%40\"),\"\\\\+\",\"%2B\") + ":${GD_PASSWORD}@${GDC_WEBDAV_HOST}");
+$out.0.fileName = $in.0.fileName;
+return ALL;
 }
 HEREDOC
+          when :s3
+            reformat_func = <<HEREDOC
 
-                builder.Phase(:number => phase += 1) do
-                  Core::build_node2(builder, Nodes.file_list2({:id => "#{id}_file_list", :name => "#{id}_file_list", :dataPolicy => "Strict", :baseURL => "#{tap[:source]}", :output_mapping => Nodes::MAP_ALL}))
-                  Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat", :id => "#{id}_reformat", :transformation => reformat_func}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat:0", :fromNode => "#{id}_file_list:0", :metadata => "file_list"}))
-                  Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_writer", :fileURL => "data/#{id}_files_to_read.csv"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_writer:0", :fromNode => "#{id}_reformat:0", :metadata => "file_list"}))
-                end
+function integer transform() {
+  
+$out.0.filePath = replace($in.0.filePath, "https://", "https://${S3_ACCESS_KEY_ID}:${S3_SECRET_ACCESS_KEY}@");
+$out.0.fileName = $in.0.fileName;
+return ALL;
+}
+HEREDOC
+          else
+            reformat_func = Nodes::MAP_ALL
+          end
+          taps.each do |tap|
+            source = tap[:source]
+            id =  tap[:id]
+
+            builder.Phase(:number => phase += 1) do
+              Core::build_node2(builder, Nodes.file_list2({:id => "#{id}_file_list", :name => "#{id}_file_list", :dataPolicy => "Strict", :baseURL => "#{tap[:source]}", :output_mapping => Nodes::MAP_ALL}))
+              Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat", :id => "#{id}_reformat", :transformation => reformat_func}))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat:0", :fromNode => "#{id}_file_list:0", :metadata => "file_list"}))
+              Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_writer", :fileURL => "data/#{id}_files_to_read.csv"}))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_writer:0", :fromNode => "#{id}_reformat:0", :metadata => "file_list"}))
+            end
 
-                builder.Phase(:number => phase += 1) do
-                  ctl = "function integer generate() {$out.0.all = \"#{id}_SKIP_LINES=0\";return OK;}"
-                  Core::build_node2(builder, Nodes.data_generator2({:name => "#{id}_generator", :id => "#{id}_generator", :generate => ctl}))
-                  Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_csv_writer", :fileURL => "#{id}_counter.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_csv_writer:0", :fromNode => "#{id}_generator:0", :metadata => "trash_metadata"}))
-                end
+            builder.Phase(:number => phase += 1) do
+              ctl = "function integer generate() {$out.0.all = \"#{id}_SKIP_LINES=0\";return OK;}"
+              Core::build_node2(builder, Nodes.data_generator2({:name => "#{id}_generator", :id => "#{id}_generator", :generate => ctl}))
+              Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_csv_writer", :fileURL => "#{id}_counter.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_csv_writer:0", :fromNode => "#{id}_generator:0", :metadata => "trash_metadata"}))
+            end
 
-               subgraph_reformat_func = <<HEREDOC
+           subgraph_reformat_func = <<HEREDOC
 
 function integer transform() {
-    $out.0.all = "graphs/#{id}_loop.grf";  
-    return ALL;
+$out.0.all = "graphs/#{id}_loop.grf";  
+return ALL;
 }
 HEREDOC
 
-            fail_reformat = <<HEREDOC
+        fail_reformat = <<HEREDOC
 function integer transform() {
-  raiseError("Loop failed");
+raiseError("Loop failed");
 }
 HEREDOC
 
 
-                builder.Phase(:number => phase += 1) do
-                  Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat graph", :id => "#{id}_reformat_graph", :transformation => subgraph_reformat_func}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_graph:0", :fromNode => "#{id}_reformat:1", :metadata => "file_list"}))
-                  Core::build_node2(builder, Nodes.run_graph2({
-                    :guiName => id,
-                    :name => id,
-                    :id => "#{id}_run_graph"
-                  }))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_run_graph:0", :fromNode => "#{id}_reformat_graph:0", :metadata => "trash_metadata"}))
-                  Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat fail", :id => "#{id}_reformat_fail", :transformation => fail_reformat}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_fail:0", :fromNode => "#{id}_run_graph:1", :metadata => "run_graph_failure_metadata"}))
-                  Core::build_node2(builder, Nodes.trash2({:name => "#{id}_trash", :id => "#{id}_trash", :debugPrint => true}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_trash:0", :fromNode => "#{id}_reformat_fail:0", :metadata => "run_graph_failure_metadata"}))
-                end
-              end
-
-
+            builder.Phase(:number => phase += 1) do
+              Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat graph", :id => "#{id}_reformat_graph", :transformation => subgraph_reformat_func}))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_graph:0", :fromNode => "#{id}_reformat:1", :metadata => "file_list"}))
+              Core::build_node2(builder, Nodes.run_graph2({
+                :guiName => id,
+                :name => id,
+                :id => "#{id}_run_graph"
+              }))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_run_graph:0", :fromNode => "#{id}_reformat_graph:0", :metadata => "trash_metadata"}))
+              Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat fail", :id => "#{id}_reformat_fail", :transformation => fail_reformat}))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_fail:0", :fromNode => "#{id}_run_graph:1", :metadata => "run_graph_failure_metadata"}))
+              Core::build_node2(builder, Nodes.trash2({:name => "#{id}_trash", :id => "#{id}_trash", :debugPrint => true}))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_trash:0", :fromNode => "#{id}_reformat_fail:0", :metadata => "run_graph_failure_metadata"}))
             end
           end
         end
@@ -689,11 +771,78 @@ HEREDOC
               :file_to_loop => "data/#{id}_files_to_read.csv",
               :graph_to_run => "graphs/#{id}_download.grf"
             })
-            create_incremental_file_downloading_graph(home + "#{tap[:id]}_download.grf", [tap], params)
+            create_incremental_file_downloading_graph(home + "#{tap[:id]}_download.grf", [tap], params.merge(:delete_after_processing => true))
 
           end
         end
 
+        def self.create_backup_restoration_run_graph(file, taps, options={})
+
+          File.open(file, "w") do |file|
+            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+            builder.Graph({
+              :name => "Run graph"
+            }) do
+              builder.Global do
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+                Helpers::property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
+                Helpers::create_trash_meta(builder)
+                Helpers::create_lookup_meta(builder)
+                Helpers::create_file_list_meta(builder)
+                Helpers::create_run_graph_failure_metadata(builder)
+
+                Core::build_node2(builder, Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
+
+              end
+
+              s3_taps = taps.map do |tap|
+                file = tap[:id]
+                # binding.pry
+                Tap.create(tap.merge({:source => "../full_#{file}.csv"}))
+              end
+              generate_file_lists(s3_taps, {
+                :builder => builder,
+                :reformat_func => :local
+              })
+            end
+          end
+        end
+
+        # Since everything that BAM touches is stored to S3 (if credentials are provided).
+        # it should be easy to restore it to any state
+        # if the timestamp is not provided in TAP it is nevertheless created so we are ensured predictable state 
+        def self.generate_backup_restoration_graph(home, project, params)
+          home = Pathname(home)
+          only = params[:only]
+
+          incremental_taps = Taps.get_incremental(project[:taps])
+
+          taps_to_process = if only.blank?
+            incremental_taps
+          else
+             temp = Taps.get_incremental([Project::find_tap_by_id(project, only)].reject {|x| x.nil?})
+             fail "There is no such tap \"#{only}\"" if temp.empty?
+             temp
+          end
+
+          # sf_taps = Taps.get_salesforce(incremental_taps)
+          # file_taps = Taps.get_file(incremental_taps)
+
+          create_backup_restoration_run_graph(home + "main.grf", taps_to_process)
+
+          taps_to_process.each do |tap|
+            id = tap[:id]
+            Helpers::loop_over_file(home + "#{tap[:id]}_loop.grf", {
+              :token => id,
+              :file_to_loop => "data/#{id}_files_to_read.csv",
+              :graph_to_run => "graphs/#{id}_download.grf"
+            })
+            # Do not backup since we are just using backup files. Also explicitely forbid deleting the files from S3 to prevent default behavior
+            restoration_params = params.merge(:s3_backup => false, :delete_after_processing => false)
+            create_incremental_file_downloading_graph(home + "#{tap[:id]}_download.grf", [tap], restoration_params)
+          end
+        end
 
         def self.generate_history_downloaders(home, project, params)
           home = Pathname(home)
@@ -799,8 +948,10 @@ HEREDOC
                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es:0", :fromNode => "#{file}_es_sort:0", :metadata => "#{file}_es_metadata"}))
                 end
 
-                builder.Phase(:number => phase += 1) do
-                  Core::build_node2(builder, Nodes.file_delete2({:guiName => "#{file}_file_delete", :name => "#{file}_file_delete", :id => "#{file}_file_delete", :baseURL => "${FILE}"}))
+                if options[:delete_after_processing] == true
+                  builder.Phase(:number => phase += 1) do
+                    Core::build_node2(builder, Nodes.file_delete2({:guiName => "#{file}_file_delete", :name => "#{file}_file_delete", :id => "#{file}_file_delete", :baseURL => "${FILE}"}))
+                  end
                 end
 
               end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gd_bam
 version: !ruby/object:Gem::Version
-  version: 0.1.35
+  version: 0.1.36
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-10 00:00:00.000000000 Z
+date: 2013-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -283,6 +283,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement