petasos 0.4.2 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 416a801bc14a8016dcf094035b3b59a4cf7739a4e7e9b473da80afa17dd9769e
4
- data.tar.gz: 77c23fe0641d5d358568671523c89f97470be8be8708a7c8efc3d05c07e75063
3
+ metadata.gz: d13f219555d9e62adf300ef1f708222955c8ebec955e64843a679624c53fc2f1
4
+ data.tar.gz: b24a1fd6cf6f055f9340d28aa06de4b7096f7a363d7c617daeaf3e2fb0838b96
5
5
  SHA512:
6
- metadata.gz: c78d7f8c16ebd748640c3b6b7170e7b63368220e53dc4285d770d86bce1cc7914c62c3e89744f350ce4e52f24827fbbcf466b0bff6e2bf27009f789e051b37d8
7
- data.tar.gz: 579ee6c2ca4e937dd6dc2bd541d35a4e5d52a4a720f7c95dee4487ee4c89e0348126acb3a68edc326bf65524c5f459db370eb104403551ca96eeb5eb6ffd9fa3
6
+ metadata.gz: 135cf3a94737ec8a8d13e824c3c20c4c4c6f7089ed6d279c9339109d5f346d877a697dded2f2f11147c375527a8ca8d3a13f15ae77a718592d121838521d0385
7
+ data.tar.gz: 385e42e07d43317eb4aac5d9082bb195ad79c664e26e5770bdb60a976f9737fe1ff99f167bcd5c57da6ddbfde06d34e6f96bd40cd1ab5b99cae238140a372c4c
data/bin/petasos CHANGED
@@ -1,5 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require 'petasos'
3
+ require "petasos"
4
4
 
5
- Petasos.new.run
5
+ Petasos.new.run(ARGV[0])
@@ -17,6 +17,11 @@ class Petasos::Distributor
17
17
  @nodes[node["name"]] = Petasos::Node.new(node)
18
18
  end
19
19
 
20
+ @nodes.values.each do |node|
21
+ puts "Running `petasos locations` on #{node.name} before distribution begins"
22
+ `ssh #{node.host} \"cd #{node.path} && petasos locations\"`
23
+ end
24
+
20
25
  @manifests = {}
21
26
  @nodes.each_pair do |node_name, node|
22
27
  @manifests[node_name] = node.manifests
@@ -50,7 +55,7 @@ class Petasos::Distributor
50
55
  @manifests.each_pair do |node_name, manifest_list|
51
56
  manifest_list.each do |manifest|
52
57
  manifest["imports"].each_pair do |pool_name, import_hash|
53
- @pools[pool_name]["import_paths"] << [node_name, import_hash["import_path"]]
58
+ @pools[pool_name]["import_paths"] << [node_name, manifest["name"], import_hash["import_path"]]
54
59
  @pools[pool_name]["backfill_import_paths"] << [node_name, manifest["name"], import_hash["import_path"]] if import_hash["backfill"]
55
60
  end
56
61
  manifest["exports"].each_pair do |pool_name, export_hash|
@@ -62,6 +67,7 @@ class Petasos::Distributor
62
67
  # {"wow-ah"=>
63
68
  # {"import_paths"=>
64
69
  # [["petasos-node-b",
70
+ # "linux-laptop-storage",
65
71
  # "/home/justin/play/petasos/test/sandbox/node_b/location_a/data"]],
66
72
  # "backfill_import_paths"=>
67
73
  # [["petasos-node-b",
@@ -77,11 +83,17 @@ class Petasos::Distributor
77
83
  export_filename = File.basename(exports_file_path, ".*")
78
84
  label, location_name, pool_name, datetime = export_filename.split("_")
79
85
  export_paths = YAML.load_file(exports_file_path)
80
- export_paths.each do |export_path|
81
- @pools[pool_name]["import_paths"].each do |pool_storage|
82
- to_node = find_node(pool_storage.first)
83
- `scp #{from_node.host}:#{export_path}* #{to_node.host}:#{pool_storage.last}`
86
+ @pools[pool_name]["import_paths"].each do |pool_storage|
87
+ to_node = find_node(pool_storage.first)
88
+ destination_seen_file_hash = get_seen_file_hash(pool_storage.first, pool_storage[1], pool_name)
89
+ export_paths.each do |export_path|
90
+ unless destination_seen_file_hash[File.basename(export_path)]
91
+ puts "moving #{File.basename(export_path)} from #{from_node.name} to #{to_node.name}"
92
+ `scp #{from_node.host}:#{export_path}* #{to_node.host}:#{pool_storage.last}`
93
+ end
84
94
  end
95
+ puts "Running `petasos locations` on #{to_node.name} after export from #{from_node.name}"
96
+ `ssh #{to_node.host} \"cd #{to_node.path} && petasos locations\"`
85
97
  end
86
98
  # mark it as completed
87
99
  completed_export_file_path = File.join(Dir.pwd, "completed-#{File.basename(exports_file_path)}")
@@ -102,39 +114,43 @@ class Petasos::Distributor
102
114
  # "canonical_exporters"=>[["petasos-node-a", "linux-laptop-source"]]}}
103
115
 
104
116
  # Process the backfills.
105
- # grab the seen files on the canonical exporters
106
117
  @pools.each_pair do |pool_name, manifest_hash|
107
- manifest_hash["canonical_exporters"].each do |canonical_exporter_details|
108
- find_node(canonical_exporter_details.first).grab_seen_file_for_location(canonical_exporter_details.last, pool_name)
109
- end
110
-
111
- # grab the seen files on the backfill importers
112
- manifest_hash["backfill_import_paths"].each do |backfill_importer_details|
113
- find_node(backfill_importer_details.first).grab_seen_file_for_location(backfill_importer_details[1], pool_name)
114
- end
115
-
116
118
  # for each canonical exporter loop through the backfill lists, identify files that need moving and move them
117
119
  manifest_hash["canonical_exporters"].each do |canonical_exporter_details|
118
- exporter_seen_files = {}
119
- exporter_file_list = YAML.load_file("seen_#{canonical_exporter_details.last}_#{pool_name}.yaml")
120
- exporter_file_list.each { |f| exporter_seen_files[File.basename(f)] = f }
120
+ exporter_seen_files = get_seen_file_hash(canonical_exporter_details.first, canonical_exporter_details.last, pool_name)
121
121
  manifest_hash["backfill_import_paths"].each do |backfill_importer_details|
122
- backfill_importer_files = {}
123
- backfill_file_list = YAML.load_file("seen_#{backfill_importer_details[1]}_#{pool_name}.yaml")
124
- backfill_file_list.each { |f| backfill_importer_files[File.basename(f)] = f }
122
+ backfill_importer_files = get_seen_file_hash(backfill_importer_details.first, backfill_importer_details[1], pool_name)
123
+
124
+ from_node = find_node(canonical_exporter_details.first)
125
+ to_node = find_node(backfill_importer_details.first)
125
126
 
126
127
  exporter_seen_files.each_pair do |file_name, file_path|
127
128
  unless backfill_importer_files[file_name]
128
- from_node = find_node(canonical_exporter_details.first)
129
- to_node = find_node(backfill_importer_details.first)
129
+ puts "Backfilling #{file_name} to #{to_node.name} from #{from_node.name}"
130
130
  `scp #{from_node.host}:#{file_path} #{to_node.host}:#{backfill_importer_details.last}`
131
131
  end
132
132
  end
133
+
134
+ puts "Running `petasos locations` on #{to_node.name} after backfill from #{from_node.name}"
135
+ `ssh #{to_node.host} \"cd #{to_node.path} && petasos locations\"`
133
136
  end
134
137
  end
135
- # clear the seen files locally.
136
- `rm seen_*`
137
138
  end
139
+ # clear the seen files locally.
140
+ `rm seen_*`
141
+ end
142
+
143
+ def get_seen_file_hash(node_name, location_name, pool_name)
144
+ find_node(node_name).grab_seen_file_for_location(location_name, pool_name)
145
+ seen_file_hash = {}
146
+ # some locations/pools do not generate a seen file.
147
+ if File.file?("seen_#{location_name}_#{pool_name}.yaml")
148
+ seen_file_list = YAML.load_file("seen_#{location_name}_#{pool_name}.yaml")
149
+ else
150
+ seen_file_list = []
151
+ end
152
+ seen_file_list.each { |f| seen_file_hash[File.basename(f)] = f }
153
+ seen_file_hash
138
154
  end
139
155
 
140
156
  def find_node(node_name)
@@ -97,7 +97,7 @@ class Petasos::Location
97
97
  pool_imports[pool["name"]]["import_path"] = File.join(pool["path"], pool_import_path)
98
98
  pool_imports[pool["name"]]["backfill"] = pool["backfill"] ? true : false
99
99
  end
100
- if pool["export"]
100
+ if pool["export"] || pool["canonical"]
101
101
  pool_exports[pool["name"]]["path"] = pool["path"]
102
102
  pool_exports[pool["name"]]["canonical"] = pool["canonical"] ? true : false
103
103
  end
data/lib/petasos/node.rb CHANGED
@@ -14,12 +14,16 @@ class Petasos::Node
14
14
  parse_manifests
15
15
  end
16
16
 
17
+ def name
18
+ config["name"]
19
+ end
20
+
17
21
  def host
18
- @config["host"]
22
+ config["host"]
19
23
  end
20
24
 
21
25
  def path
22
- @config["path"]
26
+ config["path"]
23
27
  end
24
28
 
25
29
  def grab_manifest_and_exports
data/lib/petasos.rb CHANGED
@@ -6,9 +6,11 @@ require "yaml"
6
6
  class Petasos
7
7
  class Error < StandardError; end
8
8
 
9
- def run
9
+ def run(mode = "")
10
10
  process_locations
11
- process_distribution if File.file?(File.join(Dir.pwd, "petasos_distribution-config.yaml"))
11
+ unless mode == "locations"
12
+ process_distribution if File.file?(File.join(Dir.pwd, "petasos_distribution-config.yaml"))
13
+ end
12
14
  end
13
15
 
14
16
  def process_locations
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: petasos
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.2
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Justin Myers