sushi_fabric 0.7.1 → 0.7.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/sushi_fabric/sushiApp.rb +65 -7
- data/lib/sushi_fabric/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5979375fed31ca635c84b55e996d5fc40254c3de
|
4
|
+
data.tar.gz: 9814016fb3137614502cc9a34fc2e041d6e6e85a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 42a57301a53a0bd6ea81c66302d38658e4516095853740c279e6abca13c774270f2ab3935e02be1e7aa8f25abfafc62e1f5128953383c18ff7cbf7275637bf89
|
7
|
+
data.tar.gz: 6f152feb2f8466f42c8914591059d406a4873c5030572d6dd4216168f5aa3fa192a8cf907786a3fa9f33cecdb947841fb6450ace952974d64e0e3df9785dc0c0
|
@@ -1,6 +1,6 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
# encoding: utf-8
|
3
|
-
# Version = '
|
3
|
+
# Version = '20170324-161522'
|
4
4
|
|
5
5
|
require 'csv'
|
6
6
|
require 'fileutils'
|
@@ -442,10 +442,15 @@ rm -rf #{@scratch_dir} || exit 1
|
|
442
442
|
end
|
443
443
|
job_id
|
444
444
|
end
|
445
|
-
def submit(job_script)
|
445
|
+
def submit(job_script, mock=false)
|
446
446
|
begin
|
447
|
-
job_id =
|
448
|
-
|
447
|
+
job_id = unless mock
|
448
|
+
i = submit_command(job_script)
|
449
|
+
i.to_i
|
450
|
+
else
|
451
|
+
#Time.now.to_f.to_s.gsub('.', '')
|
452
|
+
1234
|
453
|
+
end
|
449
454
|
unless job_id.to_i > 1
|
450
455
|
@logger.error("#"*50)
|
451
456
|
time = Time.now.strftime("[%Y.%m.%d %H:%M:%S]")
|
@@ -658,7 +663,7 @@ rm -rf #{@scratch_dir} || exit 1
|
|
658
663
|
data_set.id
|
659
664
|
end
|
660
665
|
end
|
661
|
-
def main
|
666
|
+
def main(mock=false)
|
662
667
|
## sushi writes creates the job scripts and builds the result data set that is to be generated
|
663
668
|
@result_dataset = []
|
664
669
|
@job_scripts = []
|
@@ -673,13 +678,18 @@ rm -rf #{@scratch_dir} || exit 1
|
|
673
678
|
warn "the process mode (#{@params['process_mode']}) is not defined"
|
674
679
|
raise "stop job submitting"
|
675
680
|
end
|
681
|
+
if mock
|
682
|
+
make_dummy_files
|
683
|
+
end
|
676
684
|
copy_inputdataset_parameter_jobscripts
|
677
685
|
|
678
686
|
# job submittion
|
687
|
+
gstore_job_script_paths = []
|
679
688
|
@job_scripts.each_with_index do |job_script, i|
|
680
|
-
if job_id = submit(job_script)
|
689
|
+
if job_id = submit(job_script, mock)
|
681
690
|
@job_ids << job_id
|
682
691
|
print "Submit job #{File.basename(job_script)} job_id=#{job_id}"
|
692
|
+
gstore_job_script_paths << File.join(@gstore_script_dir, File.basename(job_script))
|
683
693
|
end
|
684
694
|
end
|
685
695
|
|
@@ -721,9 +731,10 @@ rm -rf #{@scratch_dir} || exit 1
|
|
721
731
|
end
|
722
732
|
|
723
733
|
# save job and dataset relation in Sushi DB
|
724
|
-
job_ids.
|
734
|
+
job_ids.each_with_index do |job_id, i|
|
725
735
|
new_job = Job.new
|
726
736
|
new_job.submit_job_id = job_id.to_i
|
737
|
+
new_job.script_path = gstore_job_script_paths[i]
|
727
738
|
new_job.next_dataset_id = @next_dataset_id
|
728
739
|
new_job.save
|
729
740
|
new_job.data_set.jobs << new_job
|
@@ -756,6 +767,53 @@ rm -rf #{@scratch_dir} || exit 1
|
|
756
767
|
main
|
757
768
|
end
|
758
769
|
end
|
770
|
+
def make_dummy_files
|
771
|
+
dummy_files_header = []
|
772
|
+
headers = @result_dataset.map{|row| row.keys}.flatten.uniq
|
773
|
+
headers.select{|header| header.tag?('File')||header.tag?('Link')}.each do |header|
|
774
|
+
dummy_files_header << header
|
775
|
+
end
|
776
|
+
dummy_files_ = []
|
777
|
+
@result_dataset.each do |row|
|
778
|
+
dummy_files_.concat(dummy_files_header.map{|header| row[header]})
|
779
|
+
end
|
780
|
+
dummy_files = []
|
781
|
+
dummy_files_.each do |file|
|
782
|
+
dummy_files << file.gsub(@result_dir, '')
|
783
|
+
end
|
784
|
+
dummy_files.uniq!
|
785
|
+
|
786
|
+
dirs = []
|
787
|
+
dummy_files.permutation(2).each do |a,b|
|
788
|
+
if a.include?(b) and b !~ /\./
|
789
|
+
dirs << b
|
790
|
+
end
|
791
|
+
end
|
792
|
+
dirs.each do |dir|
|
793
|
+
dummy_files.delete(dir)
|
794
|
+
end
|
795
|
+
dirs.each do |dir|
|
796
|
+
command = "mkdir -p #{File.join(@scratch_result_dir, dir)}"
|
797
|
+
puts command
|
798
|
+
`#{command}`
|
799
|
+
end
|
800
|
+
dummy_files.each do |file|
|
801
|
+
command = if file =~ /.html/
|
802
|
+
"echo 'Hello, SUSHI world!' > #{File.join(@scratch_result_dir, file)}"
|
803
|
+
else
|
804
|
+
"touch #{File.join(@scratch_result_dir, file)}"
|
805
|
+
end
|
806
|
+
puts command
|
807
|
+
`#{command}`
|
808
|
+
end
|
809
|
+
end
|
810
|
+
def mock_run
|
811
|
+
test_run
|
812
|
+
prepare_result_dir
|
813
|
+
save_parameters_as_tsv
|
814
|
+
save_input_dataset_as_tsv
|
815
|
+
main(true)
|
816
|
+
end
|
759
817
|
def test_run
|
760
818
|
set_input_dataset
|
761
819
|
set_dir_paths
|
data/lib/sushi_fabric/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sushi_fabric
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Functional Genomics Center Zurich
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-03-
|
11
|
+
date: 2017-03-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|