rbbt-util 5.32.6 → 5.32.7

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2bc1c3443b09a4274a030b64479091e506ed1e165371d999eb154b750b2806b3
4
- data.tar.gz: 83d0ae91ee88ed3283985b5d7101759bbb45db37fbb54271bc7f16fae8ab4f52
3
+ metadata.gz: b989d145b1a34baab93f351ea9944e73dde157e9973a9443717b84a43e7b41d5
4
+ data.tar.gz: d6e8f5a7e8cb2d2f0e9a99f36a7594e4fc3655dd1d4f9b571f73f0f947b19fd3
5
5
  SHA512:
6
- metadata.gz: 0b83533d504d6d66b06083a7f19a689e9b3f82fefbc26278d9bd9af0efbdfb010f87cc609bf90396bb17257a643715e335ac8127f9ff1f11ccdd52f639b77ca8
7
- data.tar.gz: a4b93ca7d481120edde1d65f529984e652c261f365c67bea6d1c463d360733006d29486d3851d0b071ba2005cb90cf71382eb5665c7d4e1828a25f322b570ce7
6
+ metadata.gz: da60531d5e35efd54b0c4de7e9ad6b034ef05e8d58d6a6b0a1b15422b324dd6eb5a4cb01b21ba60c091616d512106c278ef794088f87f844bd96e86d8f61d7da
7
+ data.tar.gz: 2327575017d9dbb301b8c0ba36ee691bc8be00175712e2121da019251c3735d0348cae6d18e8112b00c952e233eb806a19f2275563310360ed745a9b868a6021
data/lib/rbbt/resource.rb CHANGED
@@ -313,7 +313,10 @@ url='#{url}'
313
313
  def identify(path)
314
314
  path = File.expand_path(path)
315
315
  resource ||= Rbbt
316
- (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq.each do |name|
316
+ locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys)
317
+ locations -= [:current, "current"]
318
+ locations << :current
319
+ locations.uniq.each do |name|
317
320
  pattern = resource.search_paths[name]
318
321
  next if pattern.nil?
319
322
  pattern = pattern.sub('{PWD}', Dir.pwd)
data/lib/rbbt/util/cmd.rb CHANGED
@@ -248,16 +248,26 @@ module CMD
248
248
  pid = io.pids.first
249
249
 
250
250
  line = "" if bar
251
+ starting = true
251
252
  while c = io.getc
252
- STDERR << c if Log.severity <= level
253
- line << c if bar
254
- if c == "\n"
255
- bar.process(line) if bar
253
+ if starting
256
254
  if pid
257
255
  Log.logn "STDOUT [#{pid}]: ", level
258
256
  else
259
257
  Log.logn "STDOUT: ", level
260
258
  end
259
+ starting = false
260
+ end
261
+ STDERR << c if Log.severity <= level
262
+ line << c if bar
263
+ if c == "\n"
264
+ bar.process(line) if bar
265
+ starting = true
266
+ #if pid
267
+ # Log.logn "STDOUT [#{pid}]: ", level
268
+ #else
269
+ # Log.logn "STDOUT: ", level
270
+ #end
261
271
  line = "" if bar
262
272
  end
263
273
  end
@@ -0,0 +1,118 @@
1
+ module Rbbt
2
+ def self.migrate_source_paths(path, resource = Rbbt, source = nil)
3
+ if source
4
+ lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
5
+ require 'rbbt-util'
6
+ path = "#{path}"
7
+ if Open.exists?(path)
8
+ path = #{resource.to_s}.identify(path)
9
+ else
10
+ path = Path.setup(path)
11
+ end
12
+ puts path
13
+ puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
14
+ EOF
15
+
16
+ [path, paths.collect{|p| [source, p] * ":"}, lpath]
17
+ else
18
+ if File.exists?(path)
19
+ path = resource.identify(path)
20
+ else
21
+ path = Path.setup(path)
22
+ end
23
+
24
+ [path, (path.directory? ? path.glob_all : path.find_all), path]
25
+ end
26
+ end
27
+
28
+ def self.migrate_target_path(path, search_path = 'user', resource = Rbbt, target = nil)
29
+ if target
30
+ Misc.ssh_run(target, <<-EOF).split("\n").first
31
+ require 'rbbt-util'
32
+ path = "#{path}"
33
+ resource = #{resource.to_s}
34
+ search_path = "#{search_path}"
35
+ puts resource[path].find(search_path)
36
+ EOF
37
+ else
38
+ resource[path].find(search_path)
39
+ end
40
+ end
41
+
42
+ def self.migrate_files(real_paths, target, options = {})
43
+ excludes = %w(.save .crap .source tmp filecache open-remote)
44
+ excludes += (options[:exclude] || "").split(/,\s*/)
45
+ excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
46
+
47
+ other = options[:other] || []
48
+
49
+ test_str = options[:test] ? '-nv' : ''
50
+
51
+ real_paths.each do |source_path|
52
+ if File.directory?(source_path) || source_path =~ /\/$/
53
+ source_path += "/" unless source_path[-1] == "/"
54
+ target += "/" unless target[-1] == "/"
55
+ end
56
+
57
+ next if source_path == target
58
+
59
+ if options[:target]
60
+ CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
61
+ else
62
+ Open.mkdir File.dirname(target)
63
+ end
64
+
65
+ if options[:target]
66
+ target_path = [options[:target], "'" + target + "'"] * ":"
67
+ else
68
+ target_path = "'" + target + "'"
69
+ end
70
+
71
+ TmpFile.with_file do |tmp_files|
72
+ if options[:files]
73
+ Open.write(tmp_files, options[:files] * "\n")
74
+ files_from_str = "--files-from='#{tmp_files}'"
75
+ else
76
+ files_from_str = ""
77
+ end
78
+
79
+ cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
80
+
81
+
82
+ cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]
83
+
84
+ if options[:print]
85
+ puts cmd
86
+ exit 0
87
+ else
88
+ CMD.cmd_log(cmd, :log => Log::INFO)
89
+
90
+ if options[:delete] && options[:files]
91
+ remove_files = options[:files].collect{|f| File.join(source_path, f) }
92
+ dirs = remove_files.select{|f| File.directory? f }
93
+ remove_files.each do |file|
94
+ next if dirs.include? file
95
+ Open.rm file
96
+ end
97
+ dirs.each do |dir|
98
+ FileUtils.rmdir dir if Dir.glob(dir).empty?
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
105
+
106
+
107
+ def self.migrate(path, search_path, options = {})
108
+ search_path = 'user' if search_path.nil?
109
+
110
+ resource = Rbbt
111
+
112
+ path, real_paths, lpath = migrate_source_paths(path, resource, options[:source])
113
+
114
+ target = migrate_target_path(lpath, search_path, resource, options[:target])
115
+
116
+ migrate_files(real_paths, target, options)
117
+ end
118
+ end
@@ -1,3 +1,4 @@
1
+ require 'rbbt/util/migrate'
1
2
  class Step
2
3
 
3
4
  MAIN_RSYNC_ARGS="-avztAXHP --copy-links"
@@ -126,63 +127,48 @@ class Step
126
127
  end
127
128
  end
128
129
 
129
- def self.migrate(path, search_path, options = {})
130
- resource=Rbbt
131
-
132
- orig_path = path
133
- other_rsync_args = options[:rsync]
134
-
135
- recursive = options[:recursive]
130
+ def self.migrate_source_paths(path, resource = Rbbt, source = nil, recursive = true)
136
131
  recursive = false if recursive.nil?
137
-
138
- paths = if options[:source]
139
- Misc.ssh_run(options[:source], <<-EOF).split("\n")
132
+ if source
133
+ lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
140
134
  require 'rbbt-util'
141
135
  require 'rbbt/workflow'
142
136
 
143
- path = "#{path}"
144
137
  recursive = #{ recursive.to_s }
138
+ path = "#{path}"
145
139
 
146
- if File.exists?(path)
140
+ if Open.exists?(path)
147
141
  path = #{resource.to_s}.identify(path)
148
142
  else
149
143
  path = Path.setup(path)
150
144
  end
151
145
 
152
- files = path.glob_all
153
-
146
+ files = path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p }
154
147
  files = Step.job_files_for_archive(files, recursive)
155
148
 
149
+ puts path
156
150
  puts files * "\n"
157
- EOF
158
-
159
- else
160
- if File.exists?(path)
161
- path = resource.identify(path)
162
- raise "Resource #{resource} could not identify #{orig_path}" if path.nil?
163
- else
164
- path = Path.setup(path)
165
- end
166
- files = path.glob_all
167
- files = Step.job_files_for_archive(files, recursive)
168
- files
169
- end
170
-
171
-
172
- target = if options[:target]
173
- target = Misc.ssh_run(options[:target], <<-EOF).split("\n").first
174
- require 'rbbt-util'
175
- path = "var/jobs"
176
- resource = #{resource.to_s}
177
- search_path = "#{search_path}"
178
- puts resource[path].find(search_path)
179
- EOF
180
- else
181
- resource['var/jobs'].find(search_path)
182
- end
151
+ EOF
152
+
153
+ [path, paths.collect{|p| [source, p] * ":"}, lpath]
154
+ else
155
+ path = Path.setup(path.dup)
156
+ files = path.glob_all
157
+ files = Step.job_files_for_archive(files, recursive)
158
+
159
+ [path, files, path]
160
+ end
161
+ end
162
+
163
+ def self.migrate(path, search_path, options = {})
164
+ search_path = 'user' if search_path.nil?
165
+
166
+ resource = Rbbt
167
+
168
+ path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
183
169
 
184
170
  subpath_files = {}
185
- paths.sort.each do |path|
171
+ real_paths.sort.each do |path|
186
172
  parts = path.split("/")
187
173
  subpath = parts[0..-4] * "/" + "/"
188
174
 
@@ -190,73 +176,16 @@ puts resource[path].find(search_path)
190
176
  subpath = subpath_files.keys.last
191
177
  end
192
178
 
193
- source = path[subpath.length..-1]
179
+ source = path.chars[subpath.length..-1] * ""
194
180
 
195
181
  subpath_files[subpath] ||= []
196
182
  subpath_files[subpath] << source
197
183
  end
198
184
 
199
- synced_files = []
200
- subpath_files.each do |subpath, files|
201
- if options[:target]
202
- CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
203
- else
204
- Open.mkdir File.dirname(target)
205
- end
185
+ target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
206
186
 
207
- if options[:source]
208
- source = [options[:source], subpath] * ":"
209
- else
210
- source = subpath
211
- end
212
- target = [options[:target], target] * ":" if options[:target]
213
-
214
- next if File.exists?(source) && File.exists?(target) && File.expand_path(source) == File.expand_path(target)
215
-
216
- files_and_dirs = Set.new( files )
217
- files.each do |file|
218
- synced_files << File.join(subpath, file)
219
-
220
- parts = file.split("/")[0..-2].reject{|p| p.empty?}
221
- while parts.any?
222
- files_and_dirs << parts * "/"
223
- parts.pop
224
- end
225
- end
226
-
227
- TmpFile.with_file(files_and_dirs.sort_by{|l| l.length}.to_a * "\n") do |tmp_include_file|
228
- test_str = options[:test] ? '-nv' : ''
229
-
230
- cmd = "rsync #{MAIN_RSYNC_ARGS} --progress #{test_str} --files-from='#{tmp_include_file}' #{source}/ #{target}/ #{other_rsync_args}"
231
-
232
- #cmd << " && rm -Rf #{source}" if options[:delete]
233
- if options[:print]
234
- ppp Open.read(tmp_include_file)
235
- puts cmd
236
- else
237
- CMD.cmd_log(cmd, :log => Log::INFO)
238
- end
239
- end
240
- end
241
-
242
- if options[:delete] && synced_files.any?
243
- puts Log.color :magenta, "About to erase these files:"
244
- synced_files.each do |p|
245
- puts Log.color :red, p
246
- end
247
-
248
- if options[:non_interactive]
249
- response = 'yes'
250
- else
251
- puts Log.color :magenta, "Type 'yes' if you are sure:"
252
- response = STDIN.gets.chomp
253
- end
254
-
255
- if response == 'yes'
256
- synced_files.each do |p|
257
- Open.rm p
258
- end
259
- end
187
+ subpath_files.each do |subpath, files|
188
+ Rbbt.migrate_files([subpath], target, options.merge(:files => files))
260
189
  end
261
190
  end
262
191
 
@@ -2,7 +2,7 @@
2
2
 
3
3
  require 'rbbt-util'
4
4
  require 'rbbt/util/simpleopt'
5
- require 'rbbt/workflow/remote_workflow'
5
+ require 'rbbt/util/migrate'
6
6
 
7
7
  $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
8
8
 
@@ -30,83 +30,10 @@ if options[:help]
30
30
  exit 0
31
31
  end
32
32
 
33
- #excludes = %w(.save .crap .source tmp filecache open-remote workflows apps software jobs PCAWG)
34
- excludes = %w(.save .crap .source tmp filecache open-remote)
35
- excludes += (options[:exclude] || "").split(/,\s*/)
36
- excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
37
-
38
- test_str = options[:test] ? '-nv' : ''
39
-
40
33
  path, search_path, _sep, *other = ARGV
41
34
 
42
35
  search_path = 'user' if search_path.nil?
43
- resource = Rbbt
44
-
45
- path, real_paths, lpath = if options[:source]
46
- lpath, *paths = Misc.ssh_run(options[:source], <<-EOF).split("\n")
47
- require 'rbbt-util'
48
- path = "#{path}"
49
- if Open.exists?(path)
50
- path = #{resource.to_s}.identify(path)
51
- else
52
- path = Path.setup(path)
53
- end
54
- puts path
55
- puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
56
- EOF
57
- [path, paths.collect{|p| [options[:source], p] * ":"}, lpath]
58
- else
59
- if File.exists?(path)
60
-
61
- path = resource.identify(path)
62
- else
63
- path = Path.setup(path)
64
- end
65
- [path, path.glob_all, path]
66
- end
67
-
68
- target = if options[:target]
69
- target = Misc.ssh_run(options[:target], <<-EOF).split("\n").first
70
- require 'rbbt-util'
71
- path = "#{path}"
72
- resource = #{resource.to_s}
73
- search_path = "#{search_path}"
74
- puts resource[path].find(search_path)
75
- EOF
76
- else
77
- resource[lpath].find(search_path)
78
- end
79
-
80
- real_paths.each do |source|
81
-
82
-
83
- if File.directory?(source) || source =~ /\/$/
84
- source += "/" unless source[-1] == "/"
85
- target += "/" unless target[-1] == "/"
86
- end
87
36
 
88
- next if source == target
37
+ options[:other] = other
89
38
 
90
- if options[:target]
91
- CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
92
- else
93
- Open.mkdir File.dirname(target)
94
- end
95
-
96
- if options[:target]
97
- target_path = [options[:target], "'" + target + "'"] * ":"
98
- else
99
- target_path = "'" + target + "'"
100
- end
101
-
102
- cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{excludes_str} '#{source}' #{target_path} #{other * " "}"
103
-
104
- cmd << " && rm -Rf #{source}" if options[:delete]
105
-
106
- if options[:print]
107
- puts cmd
108
- exit 0
109
- else
110
- CMD.cmd_log(cmd, :log => Log::INFO)
111
- end
112
- end
39
+ Rbbt.migrate(path, search_path, options)
@@ -0,0 +1,36 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
+ require 'rbbt-util'
3
+ require 'rbbt/util/migrate'
4
+
5
+ class TestMigrate < Test::Unit::TestCase
6
+ def _test_source_locate
7
+ assert_equal 'var/jobs/', Rbbt.migrate_source_paths(Rbbt.root['var/jobs'].find(:user)).last
8
+ assert Rbbt.migrate_source_paths(Rbbt.root['var/jobs'].find(:user))[1].include?(File.join(ENV["HOME"], '.rbbt/var/jobs/'))
9
+ end
10
+
11
+ def test_migrate
12
+ Open.rm_rf Rbbt.tmp.test.migration_test.find(:user)
13
+ test_file = Rbbt.tmp.test.migration_test.migration_test_file.find(:user)
14
+ Open.write(test_file, "TEST")
15
+ TmpFile.with_file do |tmpdir|
16
+ Misc.in_dir tmpdir do
17
+ Log.with_severity 0 do
18
+ Rbbt.migrate('tmp/test/migration_test/migration_test_file', :current)
19
+ end
20
+ end
21
+ assert_equal "TEST", Open.read(File.join(tmpdir, 'tmp/test/migration_test/migration_test_file'))
22
+ end
23
+ end
24
+
25
+ def __test_migrate_turbo
26
+ Log.with_severity 0 do
27
+ TmpFile.with_file do |tmpdir|
28
+ Misc.in_dir tmpdir do
29
+ Rbbt.migrate('etc/config', :current, :source => 'mn1')
30
+ end
31
+ assert_equal "TEST", Open.read(File.join(tmpdir, 'etc/config'))
32
+ end
33
+ end
34
+ end
35
+ end
36
+
@@ -0,0 +1,31 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
+ require 'rbbt-util'
3
+ require 'rbbt/workflow'
4
+ require 'rbbt/workflow/util/archive'
5
+
6
+ module ArchiveTestWF
7
+ extend Workflow
8
+ task :test_archive => :string do
9
+ Open.write(file(:file1), "Test file")
10
+ "TEST"
11
+ end
12
+ end
13
+
14
+ class TestClass < Test::Unit::TestCase
15
+ def test_migrate
16
+ job = ArchiveTestWF.job(:test_archive)
17
+ job.run
18
+
19
+ Log.with_severity 0 do
20
+ TmpFile.with_file do |tmpdir|
21
+ Misc.in_dir tmpdir do
22
+ Step.migrate(job.path, :current, :delete => false, :print => false)
23
+ end
24
+ assert_equal "TEST", Open.read(File.join(tmpdir, 'var/jobs/ArchiveTestWF/test_archive/Default'))
25
+ end
26
+ end
27
+
28
+
29
+ end
30
+ end
31
+
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.32.6
4
+ version: 5.32.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
@@ -282,6 +282,7 @@ files:
282
282
  - lib/rbbt/util/log/progress.rb
283
283
  - lib/rbbt/util/log/progress/report.rb
284
284
  - lib/rbbt/util/log/progress/util.rb
285
+ - lib/rbbt/util/migrate.rb
285
286
  - lib/rbbt/util/misc.rb
286
287
  - lib/rbbt/util/misc/annotated_module.rb
287
288
  - lib/rbbt/util/misc/bgzf.rb
@@ -522,6 +523,7 @@ files:
522
523
  - test/rbbt/util/test_excel2tsv.rb
523
524
  - test/rbbt/util/test_filecache.rb
524
525
  - test/rbbt/util/test_log.rb
526
+ - test/rbbt/util/test_migrate.rb
525
527
  - test/rbbt/util/test_misc.rb
526
528
  - test/rbbt/util/test_open.rb
527
529
  - test/rbbt/util/test_procpath.rb
@@ -536,6 +538,7 @@ files:
536
538
  - test/rbbt/workflow/test_schedule.rb
537
539
  - test/rbbt/workflow/test_step.rb
538
540
  - test/rbbt/workflow/test_task.rb
541
+ - test/rbbt/workflow/util/test_archive.rb
539
542
  - test/rbbt/workflow/util/test_orchestrator.rb
540
543
  - test/test_helper.rb
541
544
  homepage: http://github.com/mikisvaz/rbbt-util
@@ -583,12 +586,14 @@ test_files:
583
586
  - test/rbbt/workflow/test_doc.rb
584
587
  - test/rbbt/workflow/step/test_dependencies.rb
585
588
  - test/rbbt/workflow/test_schedule.rb
589
+ - test/rbbt/workflow/util/test_archive.rb
586
590
  - test/rbbt/workflow/util/test_orchestrator.rb
587
591
  - test/rbbt/workflow/test_task.rb
588
592
  - test/rbbt/workflow/test_step.rb
589
593
  - test/rbbt/test_tsv.rb
590
594
  - test/rbbt/test_annotations.rb
591
595
  - test/rbbt/test_knowledge_base.rb
596
+ - test/rbbt/util/test_migrate.rb
592
597
  - test/rbbt/util/test_simpleDSL.rb
593
598
  - test/rbbt/util/concurrency/processes/test_socket.rb
594
599
  - test/rbbt/util/concurrency/test_processes.rb