petasos 0.4.3 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0abd063cdacfb24b5876edc9608043694df00a102f143cc49b688b0e2678728e
-  data.tar.gz: f1393df6f212332660f6acb72cef96562270e9f326e105086ae2a8dc5bc350bc
+  metadata.gz: 4ff98d0ee0f69c09a8436007524d6650b8ed5e82c4f2ced088b0a8da72ac31cd
+  data.tar.gz: 4e16bc991db6f87baa0f056e64834251749c290b3e6aba9db66586ff6850a2fb
 SHA512:
-  metadata.gz: ef0d35c61eb1de183ea9638a7ac526bf18e87d48cef269eba38ab369a510f9c6d452d058f489dd673916eebf8876e287b79846c82d217e925f449cf3642a9ef8
-  data.tar.gz: 2c6d55f20670cb8000f9d26f22eb5838597d13fe92db650c84db57b3b17a485aea9bea8792f7abd45f39b27562bb9971f6c5c145e284ae6acefd0de7e6f8addc
+  metadata.gz: d0cc807729014a25f679cfadddcc8ee6851b2312438790e193ebd4b38ce9182d1a4ee4a0d0da07685eee1b125671841f1b7758b1708699d6b2328bd8ea74bab8
+  data.tar.gz: 9b44f27902559d5b0d3df48ab822186d3de4f28627b86a0f1288000e4d1e61ad7985348ea6efaf03bb94d9ba654941b48ec72a2febe6d851d5825b83753c3f46

data/bin/petasos

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'petasos'
+require "petasos"
 
-Petasos.new.run
+Petasos.new.run(ARGV[0])

data/lib/petasos/distributor.rb

@@ -50,7 +50,7 @@ class Petasos::Distributor
     @manifests.each_pair do |node_name, manifest_list|
       manifest_list.each do |manifest|
         manifest["imports"].each_pair do |pool_name, import_hash|
-          @pools[pool_name]["import_paths"] << [node_name, import_hash["import_path"]]
+          @pools[pool_name]["import_paths"] << [node_name, manifest["name"], import_hash["import_path"]]
           @pools[pool_name]["backfill_import_paths"] << [node_name, manifest["name"], import_hash["import_path"]] if import_hash["backfill"]
         end
         manifest["exports"].each_pair do |pool_name, export_hash|
@@ -62,6 +62,7 @@ class Petasos::Distributor
     # {"wow-ah"=>
     #     {"import_paths"=>
     #       [["petasos-node-b",
+    #         "linux-laptop-storage",
     #         "/home/justin/play/petasos/test/sandbox/node_b/location_a/data"]],
     #      "backfill_import_paths"=>
     #       [["petasos-node-b",
@@ -77,11 +78,17 @@ class Petasos::Distributor
       export_filename = File.basename(exports_file_path, ".*")
       label, location_name, pool_name, datetime = export_filename.split("_")
       export_paths = YAML.load_file(exports_file_path)
-      export_paths.each do |export_path|
-        @pools[pool_name]["import_paths"].each do |pool_storage|
-          to_node = find_node(pool_storage.first)
-          `scp #{from_node.host}:#{export_path}* #{to_node.host}:#{pool_storage.last}`
+      @pools[pool_name]["import_paths"].each do |pool_storage|
+        to_node = find_node(pool_storage.first)
+        destination_seen_file_hash = get_seen_file_hash(pool_storage.first, pool_storage[1], pool_name)
+        export_paths.each do |export_path|
+          unless destination_seen_file_hash[File.basename(export_path)]
+            puts "exporting #{File.basename(export_path)} from #{from_node.name} to #{to_node.name}"
+            `scp #{from_node.host}:#{export_path}* #{to_node.host}:#{pool_storage.last}`
+          end
         end
+        puts "Running `petasos locations` on #{to_node.name} after export from #{from_node.name}"
+        `ssh #{to_node.host} \"cd #{to_node.path} && petasos locations\"`
       end
       # mark it as completed
       completed_export_file_path = File.join(Dir.pwd, "completed-#{File.basename(exports_file_path)}")
@@ -89,6 +96,8 @@ class Petasos::Distributor
       # and then put it back where it came from
       `scp #{completed_export_file_path} #{from_node.host}:#{from_node.path}`
       `rm #{completed_export_file_path}`
+      puts "Running `petasos locations` on #{from_node.name} after completing its exports"
+      `ssh #{from_node.host} \"cd #{from_node.path} && petasos locations\"`
     end
 
     # {"wow-ah"=>
@@ -102,39 +111,43 @@ class Petasos::Distributor
     #      "canonical_exporters"=>[["petasos-node-a", "linux-laptop-source"]]}}
 
     # Process the backfills.
-    # grab the seen files on the canonical exporters
     @pools.each_pair do |pool_name, manifest_hash|
-      manifest_hash["canonical_exporters"].each do |canonical_exporter_details|
-        find_node(canonical_exporter_details.first).grab_seen_file_for_location(canonical_exporter_details.last, pool_name)
-      end
-
-      # grab the seen files on the backfill importers
-      manifest_hash["backfill_import_paths"].each do |backfill_importer_details|
-        find_node(backfill_importer_details.first).grab_seen_file_for_location(backfill_importer_details[1], pool_name)
-      end
-
       # for each canonical exporter loop through the backfill lists, identify files that need moving and move them
       manifest_hash["canonical_exporters"].each do |canonical_exporter_details|
-        exporter_seen_files = {}
-        exporter_file_list = YAML.load_file("seen_#{canonical_exporter_details.last}_#{pool_name}.yaml")
-        exporter_file_list.each { |f| exporter_seen_files[File.basename(f)] = f }
+        exporter_seen_files = get_seen_file_hash(canonical_exporter_details.first, canonical_exporter_details.last, pool_name)
         manifest_hash["backfill_import_paths"].each do |backfill_importer_details|
-          backfill_importer_files = {}
-          backfill_file_list = YAML.load_file("seen_#{backfill_importer_details[1]}_#{pool_name}.yaml")
-          backfill_file_list.each { |f| backfill_importer_files[File.basename(f)] = f }
+          backfill_importer_files = get_seen_file_hash(backfill_importer_details.first, backfill_importer_details[1], pool_name)
+
+          from_node = find_node(canonical_exporter_details.first)
+          to_node = find_node(backfill_importer_details.first)
 
           exporter_seen_files.each_pair do |file_name, file_path|
             unless backfill_importer_files[file_name]
-              from_node = find_node(canonical_exporter_details.first)
-              to_node = find_node(backfill_importer_details.first)
+              puts "Backfilling #{file_name} to #{to_node.name} from #{from_node.name}"
               `scp #{from_node.host}:#{file_path} #{to_node.host}:#{backfill_importer_details.last}`
             end
           end
+
+          puts "Running `petasos locations` on #{to_node.name} after backfill from #{from_node.name}"
+          `ssh #{to_node.host} \"cd #{to_node.path} && petasos locations\"`
         end
       end
-      # clear the seen files locally.
-      `rm seen_*`
     end
+    # clear the seen files locally.
+    `rm seen_*`
+  end
+
+  def get_seen_file_hash(node_name, location_name, pool_name)
+    find_node(node_name).grab_seen_file_for_location(location_name, pool_name)
+    seen_file_hash = {}
+    # some locations/pools do not generate a seen file.
+    if File.file?("seen_#{location_name}_#{pool_name}.yaml")
+      seen_file_list = YAML.load_file("seen_#{location_name}_#{pool_name}.yaml")
+    else
+      seen_file_list = []
+    end
+    seen_file_list.each { |f| seen_file_hash[File.basename(f)] = f }
+    seen_file_hash
   end
 
   def find_node(node_name)

data/lib/petasos/node.rb

@@ -10,16 +10,24 @@ class Petasos::Node
     @config = config
     @manifests = []
     `mkdir -p #{config["name"]}`
+
+    puts "Running `petasos locations` on #{name} before distribution begins"
+    `ssh #{host} \"cd #{path} && petasos locations\"`
+
     grab_manifest_and_exports
     parse_manifests
   end
 
+  def name
+    config["name"]
+  end
+
   def host
-    @config["host"]
+    config["host"]
   end
 
   def path
-    @config["path"]
+    config["path"]
   end
 
   def grab_manifest_and_exports

data/lib/petasos.rb

@@ -6,9 +6,16 @@ require "yaml"
 class Petasos
   class Error < StandardError; end
 
-  def run
-    process_locations
-    process_distribution if File.file?(File.join(Dir.pwd, "petasos_distribution-config.yaml"))
+  def run(mode = "")
+    unless File.file?("petasos_is_running")
+      `touch petasos_is_running`
+      process_locations
+      unless mode == "locations"
+        process_distribution if File.file?(File.join(Dir.pwd, "petasos_distribution-config.yaml"))
+        process_locations
+      end
+      `rm petasos_is_running`
+    end
   end
 
   def process_locations

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: petasos
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.1
 platform: ruby
 authors:
 - Justin Myers