petasos 0.5.0 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d13f219555d9e62adf300ef1f708222955c8ebec955e64843a679624c53fc2f1
-  data.tar.gz: b24a1fd6cf6f055f9340d28aa06de4b7096f7a363d7c617daeaf3e2fb0838b96
+  metadata.gz: cb623c372e4177df95553923e61d111337bb01b28ace724d443d2a2014631ad2
+  data.tar.gz: b914c31bbd9c6743f662c051b881ca9e9195542e5ec6b7b1c8890b33496470a6
 SHA512:
-  metadata.gz: 135cf3a94737ec8a8d13e824c3c20c4c4c6f7089ed6d279c9339109d5f346d877a697dded2f2f11147c375527a8ca8d3a13f15ae77a718592d121838521d0385
-  data.tar.gz: 385e42e07d43317eb4aac5d9082bb195ad79c664e26e5770bdb60a976f9737fe1ff99f167bcd5c57da6ddbfde06d34e6f96bd40cd1ab5b99cae238140a372c4c
+  metadata.gz: 8882b063964e50f3c3e6109d652eedac094c619007d173760ccc769c2452b86ab875b1355c5434d431e20a1e634624dd642207484f76d1c13de6bcc96ceaa03d
+  data.tar.gz: 5e545b2c1fbe779f1effa4641fe3fa93b5f46aa60611bacaae6e24d9b57a5809fe5bac79c17ab5f996dfa80a62dfe2258b6289faa3ba9a817653b81f43e1adfd

data/lib/petasos/distributor.rb

@@ -17,11 +17,6 @@ class Petasos::Distributor
       @nodes[node["name"]] = Petasos::Node.new(node)
     end
 
-    @nodes.values.each do |node|
-      puts "Running `petasos locations` on #{node.name} before distribution begins"
-      `ssh #{node.host} \"cd #{node.path} && petasos locations\"`
-    end
-
     @manifests = {}
     @nodes.each_pair do |node_name, node|
       @manifests[node_name] = node.manifests
@@ -88,7 +83,7 @@ class Petasos::Distributor
         destination_seen_file_hash = get_seen_file_hash(pool_storage.first, pool_storage[1], pool_name)
         export_paths.each do |export_path|
           unless destination_seen_file_hash[File.basename(export_path)]
-            puts "moving #{File.basename(export_path)} from #{from_node.name} to #{to_node.name}"
+            puts "exporting #{File.basename(export_path)} from #{from_node.name} to #{to_node.name}"
             `scp #{from_node.host}:#{export_path}* #{to_node.host}:#{pool_storage.last}`
           end
         end
@@ -101,6 +96,8 @@ class Petasos::Distributor
       # and then put it back where it came from
       `scp #{completed_export_file_path} #{from_node.host}:#{from_node.path}`
       `rm #{completed_export_file_path}`
+      puts "Running `petasos locations` on #{from_node.name} after completing its exports"
+      `ssh #{from_node.host} \"cd #{from_node.path} && petasos locations\"`
     end
 
     # {"wow-ah"=>
@@ -115,6 +112,7 @@ class Petasos::Distributor
 
     # Process the backfills.
     @pools.each_pair do |pool_name, manifest_hash|
+      `mkdir -p petasos_distributor_workspace`
       # for each canonical exporter loop through the backfill lists, identify files that need moving and move them
       manifest_hash["canonical_exporters"].each do |canonical_exporter_details|
         exporter_seen_files = get_seen_file_hash(canonical_exporter_details.first, canonical_exporter_details.last, pool_name)
@@ -137,15 +135,15 @@ class Petasos::Distributor
       end
     end
     # clear the seen files locally.
-    `rm seen_*`
+    `rm petasos_distributor_workspace/*`
   end
 
   def get_seen_file_hash(node_name, location_name, pool_name)
     find_node(node_name).grab_seen_file_for_location(location_name, pool_name)
     seen_file_hash = {}
     # some locations/pools do not generate a seen file.
-    if File.file?("seen_#{location_name}_#{pool_name}.yaml")
-      seen_file_list = YAML.load_file("seen_#{location_name}_#{pool_name}.yaml")
+    if File.file?("petasos_distributor_workspace/seen_#{location_name}_#{pool_name}.yaml")
+      seen_file_list = YAML.load_file("petasos_distributor_workspace/seen_#{location_name}_#{pool_name}.yaml")
     else
       seen_file_list = []
     end

data/lib/petasos/location.rb

@@ -144,12 +144,12 @@ class Petasos::Location
     end
   end
 
-  def clear_all_seen_pool_files
-    pools.each do |pool|
-      yaml_path = File.join(path, "seen_#{pool["name"]}.yaml")
-      write_yaml(yaml_path, [])
-    end
-  end
+  # def clear_all_seen_pool_files
+  #   pools.each do |pool|
+  #     yaml_path = File.join(path, "seen_#{pool["name"]}.yaml")
+  #     write_yaml(yaml_path, [])
+  #   end
+  # end
 
   def create_file_export_list(pool, file_paths)
     yaml_path = File.join(Dir.pwd, "exports_#{@config["name"]}_#{pool["name"]}_#{Time.now.strftime("%Y-%m-%d-%H:%M:%S")}.yaml")

data/lib/petasos/node.rb

@@ -10,6 +10,10 @@ class Petasos::Node
     @config = config
     @manifests = []
     `mkdir -p #{config["name"]}`
+
+    puts "Running `petasos locations` on #{name} before distribution begins"
+    `ssh #{host} \"cd #{path} && petasos locations\"`
+
     grab_manifest_and_exports
     parse_manifests
   end
@@ -35,7 +39,7 @@ class Petasos::Node
   end
 
   def grab_seen_file_for_location(location_name, pool_name)
-    `scp #{config["host"]}:#{config["path"]}/seen_#{location_name}_#{pool_name}.yaml .`
+    `scp #{config["host"]}:#{config["path"]}/seen_#{location_name}_#{pool_name}.yaml petasos_distributor_workspace/`
   end
 
   def parse_manifests

data/lib/petasos.rb

@@ -7,9 +7,14 @@ class Petasos
   class Error < StandardError; end
 
   def run(mode = "")
-    process_locations
-    unless mode == "locations"
-      process_distribution if File.file?(File.join(Dir.pwd, "petasos_distribution-config.yaml"))
+    unless File.file?("petasos_is_running")
+      `touch petasos_is_running`
+      process_locations
+      unless mode == "locations"
+        process_distribution if File.file?(File.join(Dir.pwd, "petasos_distribution-config.yaml"))
+        process_locations
+      end
+      `rm petasos_is_running`
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: petasos
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.2
 platform: ruby
 authors:
 - Justin Myers