miga-base 0.7.4.0 → 0.7.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/miga/cli.rb +10 -8
- data/lib/miga/cli/action.rb +2 -3
- data/lib/miga/cli/action/about.rb +5 -6
- data/lib/miga/cli/action/add.rb +18 -12
- data/lib/miga/cli/action/add_result.rb +2 -3
- data/lib/miga/cli/action/archive.rb +1 -2
- data/lib/miga/cli/action/classify_wf.rb +8 -6
- data/lib/miga/cli/action/console.rb +0 -1
- data/lib/miga/cli/action/daemon.rb +7 -7
- data/lib/miga/cli/action/date.rb +0 -1
- data/lib/miga/cli/action/derep_wf.rb +5 -4
- data/lib/miga/cli/action/doctor.rb +28 -20
- data/lib/miga/cli/action/doctor/base.rb +29 -6
- data/lib/miga/cli/action/edit.rb +1 -2
- data/lib/miga/cli/action/files.rb +8 -8
- data/lib/miga/cli/action/find.rb +5 -6
- data/lib/miga/cli/action/generic.rb +7 -7
- data/lib/miga/cli/action/get.rb +20 -17
- data/lib/miga/cli/action/get_db.rb +8 -2
- data/lib/miga/cli/action/index_wf.rb +1 -1
- data/lib/miga/cli/action/init.rb +34 -29
- data/lib/miga/cli/action/init/daemon_helper.rb +65 -43
- data/lib/miga/cli/action/lair.rb +7 -7
- data/lib/miga/cli/action/ln.rb +6 -6
- data/lib/miga/cli/action/ls.rb +1 -2
- data/lib/miga/cli/action/ncbi_get.rb +11 -3
- data/lib/miga/cli/action/new.rb +4 -4
- data/lib/miga/cli/action/next_step.rb +0 -1
- data/lib/miga/cli/action/preproc_wf.rb +3 -3
- data/lib/miga/cli/action/quality_wf.rb +1 -1
- data/lib/miga/cli/action/rm.rb +2 -3
- data/lib/miga/cli/action/run.rb +8 -8
- data/lib/miga/cli/action/stats.rb +3 -3
- data/lib/miga/cli/action/summary.rb +7 -6
- data/lib/miga/cli/action/tax_dist.rb +8 -4
- data/lib/miga/cli/action/tax_index.rb +3 -4
- data/lib/miga/cli/action/tax_set.rb +7 -6
- data/lib/miga/cli/action/tax_test.rb +6 -5
- data/lib/miga/cli/action/wf.rb +21 -19
- data/lib/miga/cli/base.rb +34 -32
- data/lib/miga/cli/objects_helper.rb +24 -17
- data/lib/miga/cli/opt_helper.rb +3 -2
- data/lib/miga/common.rb +2 -5
- data/lib/miga/common/base.rb +15 -16
- data/lib/miga/common/format.rb +8 -5
- data/lib/miga/common/hooks.rb +1 -4
- data/lib/miga/common/path.rb +4 -9
- data/lib/miga/common/with_daemon.rb +5 -2
- data/lib/miga/common/with_daemon_class.rb +1 -1
- data/lib/miga/common/with_result.rb +2 -1
- data/lib/miga/daemon.rb +51 -35
- data/lib/miga/daemon/base.rb +0 -2
- data/lib/miga/dataset.rb +47 -37
- data/lib/miga/dataset/base.rb +52 -37
- data/lib/miga/dataset/hooks.rb +3 -4
- data/lib/miga/dataset/result.rb +17 -1
- data/lib/miga/json.rb +5 -7
- data/lib/miga/lair.rb +4 -0
- data/lib/miga/metadata.rb +4 -3
- data/lib/miga/project.rb +29 -20
- data/lib/miga/project/base.rb +52 -37
- data/lib/miga/project/dataset.rb +27 -13
- data/lib/miga/project/hooks.rb +0 -3
- data/lib/miga/project/result.rb +14 -5
- data/lib/miga/remote_dataset.rb +85 -72
- data/lib/miga/remote_dataset/base.rb +11 -13
- data/lib/miga/remote_dataset/download.rb +33 -12
- data/lib/miga/result.rb +34 -25
- data/lib/miga/result/base.rb +0 -2
- data/lib/miga/result/dates.rb +1 -3
- data/lib/miga/result/source.rb +15 -16
- data/lib/miga/result/stats.rb +36 -25
- data/lib/miga/tax_dist.rb +6 -3
- data/lib/miga/tax_index.rb +17 -17
- data/lib/miga/taxonomy.rb +6 -1
- data/lib/miga/taxonomy/base.rb +19 -15
- data/lib/miga/version.rb +19 -16
- data/test/common_test.rb +3 -11
- data/test/daemon_helper.rb +38 -0
- data/test/daemon_test.rb +73 -101
- data/test/dataset_test.rb +58 -59
- data/test/format_test.rb +3 -11
- data/test/hook_test.rb +50 -55
- data/test/json_test.rb +7 -8
- data/test/lair_test.rb +22 -28
- data/test/metadata_test.rb +6 -14
- data/test/project_test.rb +33 -39
- data/test/remote_dataset_test.rb +20 -28
- data/test/result_stats_test.rb +17 -27
- data/test/result_test.rb +41 -34
- data/test/tax_dist_test.rb +0 -2
- data/test/tax_index_test.rb +4 -10
- data/test/taxonomy_test.rb +7 -9
- data/test/test_helper.rb +42 -1
- data/test/with_daemon_test.rb +14 -22
- data/utils/cleanup-databases.rb +6 -5
- data/utils/distance/base.rb +0 -1
- data/utils/distance/commands.rb +19 -12
- data/utils/distance/database.rb +24 -21
- data/utils/distance/pipeline.rb +12 -9
- data/utils/distance/runner.rb +14 -13
- data/utils/distance/temporal.rb +1 -3
- data/utils/distances.rb +1 -1
- data/utils/domain-ess-genes.rb +7 -7
- data/utils/index_metadata.rb +4 -2
- data/utils/mytaxa_scan.rb +18 -16
- data/utils/representatives.rb +5 -4
- data/utils/requirements.txt +1 -1
- data/utils/subclade/base.rb +0 -1
- data/utils/subclade/pipeline.rb +7 -6
- data/utils/subclade/runner.rb +9 -9
- data/utils/subclade/temporal.rb +0 -2
- data/utils/subclades-compile.rb +39 -37
- data/utils/subclades.rb +1 -1
- metadata +3 -2
data/lib/miga/json.rb
CHANGED
@@ -6,9 +6,7 @@ require 'json'
|
|
6
6
|
##
|
7
7
|
# Taxonomic classifications in MiGA.
|
8
8
|
class MiGA::Json < MiGA::MiGA
|
9
|
-
|
10
9
|
class << self
|
11
|
-
|
12
10
|
##
|
13
11
|
# Default parsing options. Supported +opts+ keys:
|
14
12
|
# - +:contents+: If true, the input is assumed to be the contents to parse,
|
@@ -25,6 +23,7 @@ class MiGA::Json < MiGA::MiGA
|
|
25
23
|
if opts[:additions] and opts[:symbolize]
|
26
24
|
raise 'JSON additions are not supported with symbolized names'
|
27
25
|
end
|
26
|
+
|
28
27
|
opts
|
29
28
|
end
|
30
29
|
|
@@ -37,12 +36,13 @@ class MiGA::Json < MiGA::MiGA
|
|
37
36
|
opts = default_opts(opts)
|
38
37
|
cont = opts[:contents] ? path : File.read(path)
|
39
38
|
raise "Empty descriptor: #{opts[:contents] ? "''" : path}." if cont.empty?
|
39
|
+
|
40
40
|
y = JSON.parse(cont,
|
41
|
-
|
42
|
-
|
41
|
+
symbolize_names: opts[:symbolize],
|
42
|
+
create_additions: opts[:additions])
|
43
43
|
unless opts[:default].nil?
|
44
44
|
opts[:default] = parse(opts[:default]) if opts[:default].is_a? String
|
45
|
-
y.each{ |k, v| opts[:default][k] = v }
|
45
|
+
y.each { |k, v| opts[:default][k] = v }
|
46
46
|
y = opts[:default]
|
47
47
|
end
|
48
48
|
y
|
@@ -56,7 +56,5 @@ class MiGA::Json < MiGA::MiGA
|
|
56
56
|
File.open(path, 'w') { |fh| fh.print y } unless path.nil?
|
57
57
|
y
|
58
58
|
end
|
59
|
-
|
60
59
|
end
|
61
|
-
|
62
60
|
end
|
data/lib/miga/lair.rb
CHANGED
@@ -70,6 +70,7 @@ class MiGA::Lair < MiGA::MiGA
|
|
70
70
|
def daemon_loop
|
71
71
|
check_directories
|
72
72
|
return false if options[:dry]
|
73
|
+
|
73
74
|
sleep(options[:latency])
|
74
75
|
true
|
75
76
|
end
|
@@ -102,10 +103,12 @@ class MiGA::Lair < MiGA::MiGA
|
|
102
103
|
def each_project(dir = path)
|
103
104
|
Dir.entries(dir).each do |f|
|
104
105
|
next if %w[. ..].include?(f) # Ruby <= 2.3 doesn't have Dir.children
|
106
|
+
|
105
107
|
f = File.join(dir, f)
|
106
108
|
if MiGA::Project.exist? f
|
107
109
|
project = MiGA::Project.load(f)
|
108
110
|
raise "Cannot load project: #{f}" if project.nil?
|
111
|
+
|
109
112
|
yield(project)
|
110
113
|
elsif Dir.exist? f
|
111
114
|
each_project(f) { |project| yield(project) }
|
@@ -127,6 +130,7 @@ class MiGA::Lair < MiGA::MiGA
|
|
127
130
|
each_project do |project|
|
128
131
|
d = MiGA::Daemon.new(project)
|
129
132
|
next if d.active?
|
133
|
+
|
130
134
|
l_alive = d.last_alive
|
131
135
|
unless l_alive.nil?
|
132
136
|
next if options[:trust_timestamp] && project.metadata.updated < l_alive
|
data/lib/miga/metadata.rb
CHANGED
@@ -16,6 +16,7 @@ class MiGA::Metadata < MiGA::MiGA
|
|
16
16
|
# exists, or nil otherwise.
|
17
17
|
def self.load(path)
|
18
18
|
return nil unless Metadata.exist? path
|
19
|
+
|
19
20
|
MiGA::Metadata.new(path)
|
20
21
|
end
|
21
22
|
|
@@ -33,11 +34,11 @@ class MiGA::Metadata < MiGA::MiGA
|
|
33
34
|
@path = File.absolute_path(path)
|
34
35
|
unless File.exist? path
|
35
36
|
@data = {}
|
36
|
-
defaults.each { |k,v| self[k] = v }
|
37
|
+
defaults.each { |k, v| self[k] = v }
|
37
38
|
create
|
38
39
|
end
|
39
40
|
end
|
40
|
-
|
41
|
+
|
41
42
|
##
|
42
43
|
# Parsed data as a Hash
|
43
44
|
def data
|
@@ -73,6 +74,7 @@ class MiGA::Metadata < MiGA::MiGA
|
|
73
74
|
ofh.close
|
74
75
|
raise "Lock-racing detected for #{path}" unless
|
75
76
|
File.exist?("#{path}.tmp") and File.exist?(lock_file)
|
77
|
+
|
76
78
|
File.rename("#{path}.tmp", path)
|
77
79
|
File.unlink(lock_file)
|
78
80
|
end
|
@@ -152,5 +154,4 @@ class MiGA::Metadata < MiGA::MiGA
|
|
152
154
|
def to_json
|
153
155
|
MiGA::Json.generate(data)
|
154
156
|
end
|
155
|
-
|
156
157
|
end
|
data/lib/miga/project.rb
CHANGED
@@ -7,19 +7,18 @@ require 'miga/project/dataset'
|
|
7
7
|
require 'miga/project/hooks'
|
8
8
|
|
9
9
|
##
|
10
|
-
# MiGA representation of a project
|
10
|
+
# MiGA representation of a project
|
11
11
|
class MiGA::Project < MiGA::MiGA
|
12
|
-
|
13
12
|
include MiGA::Project::Result
|
14
13
|
include MiGA::Project::Dataset
|
15
14
|
include MiGA::Project::Hooks
|
16
15
|
|
17
16
|
##
|
18
|
-
# Absolute path to the project folder
|
17
|
+
# Absolute path to the project folder
|
19
18
|
attr_reader :path
|
20
19
|
|
21
20
|
##
|
22
|
-
# Information about the project as MiGA::Metadata
|
21
|
+
# Information about the project as MiGA::Metadata
|
23
22
|
attr_reader :metadata
|
24
23
|
|
25
24
|
##
|
@@ -29,7 +28,7 @@ class MiGA::Project < MiGA::MiGA
|
|
29
28
|
##
|
30
29
|
# Create a new MiGA::Project at +path+, if it doesn't exist and +update+ is
|
31
30
|
# false, or load an existing one.
|
32
|
-
def initialize(path, update=false)
|
31
|
+
def initialize(path, update = false)
|
33
32
|
@datasets = {}
|
34
33
|
@do_not_save = false
|
35
34
|
@path = File.absolute_path(path)
|
@@ -40,17 +39,19 @@ class MiGA::Project < MiGA::MiGA
|
|
40
39
|
end
|
41
40
|
|
42
41
|
##
|
43
|
-
# Create an empty project
|
42
|
+
# Create an empty project
|
44
43
|
def create
|
45
44
|
unless MiGA::MiGA.initialized?
|
46
45
|
raise 'Impossible to create project in uninitialized MiGA.'
|
47
46
|
end
|
48
|
-
|
49
|
-
|
50
|
-
|
47
|
+
|
48
|
+
dirs = [path] + @@FOLDERS.map { |d| "#{path}/#{d}" } +
|
49
|
+
@@DATA_FOLDERS.map { |d| "#{path}/data/#{d}" }
|
50
|
+
dirs.each { |d| Dir.mkdir(d) unless Dir.exist? d }
|
51
51
|
@metadata = MiGA::Metadata.new(
|
52
52
|
File.expand_path('miga.project.json', path),
|
53
|
-
{datasets: [], name: File.basename(path)}
|
53
|
+
{ datasets: [], name: File.basename(path) }
|
54
|
+
)
|
54
55
|
d_path = File.expand_path('daemon/daemon.json', path)
|
55
56
|
File.open(d_path, 'w') { |fh| fh.puts '{}' } unless File.exist? d_path
|
56
57
|
pull_hook :on_create
|
@@ -58,13 +59,13 @@ class MiGA::Project < MiGA::MiGA
|
|
58
59
|
end
|
59
60
|
|
60
61
|
##
|
61
|
-
# Save any changes persistently. Do nothing if +do_not_save+ is true
|
62
|
+
# Save any changes persistently. Do nothing if +do_not_save+ is true
|
62
63
|
def save
|
63
64
|
save! unless do_not_save
|
64
65
|
end
|
65
66
|
|
66
67
|
##
|
67
|
-
# Save any changes persistently, regardless of +do_not_save
|
68
|
+
# Save any changes persistently, regardless of +do_not_save+
|
68
69
|
def save!
|
69
70
|
metadata.save
|
70
71
|
pull_hook :on_save
|
@@ -72,29 +73,37 @@ class MiGA::Project < MiGA::MiGA
|
|
72
73
|
end
|
73
74
|
|
74
75
|
##
|
75
|
-
# (Re-)load project data and metadata
|
76
|
+
# (Re-)load project data and metadata
|
76
77
|
def load
|
77
78
|
@datasets = {}
|
78
79
|
@dataset_names_hash = nil
|
79
80
|
@metadata = MiGA::Metadata.load "#{path}/miga.project.json"
|
80
81
|
raise "Couldn't find project metadata at #{path}" if metadata.nil?
|
82
|
+
|
81
83
|
pull_hook :on_load
|
82
84
|
end
|
83
85
|
|
84
86
|
##
|
85
|
-
# Name of the project
|
86
|
-
def name
|
87
|
+
# Name of the project
|
88
|
+
def name
|
89
|
+
metadata[:name]
|
90
|
+
end
|
87
91
|
|
88
92
|
##
|
89
|
-
# Type of project
|
90
|
-
def type
|
93
|
+
# Type of project
|
94
|
+
def type
|
95
|
+
metadata[:type]
|
96
|
+
end
|
91
97
|
|
92
98
|
##
|
93
99
|
# Is this a clade project?
|
94
|
-
def is_clade?
|
100
|
+
def is_clade?
|
101
|
+
type == :clade
|
102
|
+
end
|
95
103
|
|
96
104
|
##
|
97
105
|
# Is this a project for multi-organism datasets?
|
98
|
-
def is_multi?
|
99
|
-
|
106
|
+
def is_multi?
|
107
|
+
@@KNOWN_TYPES[type][:multi]
|
108
|
+
end
|
100
109
|
end
|
data/lib/miga/project/base.rb
CHANGED
@@ -2,7 +2,6 @@
|
|
2
2
|
# @license Artistic-2.0
|
3
3
|
|
4
4
|
class MiGA::Project < MiGA::MiGA
|
5
|
-
|
6
5
|
class << self
|
7
6
|
##
|
8
7
|
# Does the project at +path+ exist?
|
@@ -15,26 +14,35 @@ class MiGA::Project < MiGA::MiGA
|
|
15
14
|
# otherwise.
|
16
15
|
def load(path)
|
17
16
|
return nil unless exist? path
|
17
|
+
|
18
18
|
new path
|
19
19
|
end
|
20
20
|
|
21
|
-
def INCLADE_TASKS
|
22
|
-
|
23
|
-
|
24
|
-
def RESULT_DIRS ; @@RESULT_DIRS ; end
|
21
|
+
def INCLADE_TASKS
|
22
|
+
@@INCLADE_TASKS
|
23
|
+
end
|
25
24
|
|
26
|
-
|
25
|
+
def DISTANCE_TASKS
|
26
|
+
@@DISTANCE_TASKS
|
27
|
+
end
|
28
|
+
|
29
|
+
def KNOWN_TYPES
|
30
|
+
@@KNOWN_TYPES
|
31
|
+
end
|
27
32
|
|
33
|
+
def RESULT_DIRS
|
34
|
+
@@RESULT_DIRS
|
35
|
+
end
|
36
|
+
end
|
28
37
|
end
|
29
38
|
|
30
39
|
module MiGA::Project::Base
|
31
|
-
|
32
40
|
##
|
33
|
-
# Top-level folders inside a project
|
41
|
+
# Top-level folders inside a project
|
34
42
|
@@FOLDERS = %w[data metadata daemon]
|
35
43
|
|
36
44
|
##
|
37
|
-
# Folders for results
|
45
|
+
# Folders for results
|
38
46
|
@@DATA_FOLDERS = %w[
|
39
47
|
01.raw_reads 02.trimmed_reads 03.read_quality 04.trimmed_fasta
|
40
48
|
05.assembly 06.cds
|
@@ -54,46 +62,53 @@ module MiGA::Project::Base
|
|
54
62
|
]
|
55
63
|
|
56
64
|
##
|
57
|
-
# Directories containing the results from project-wide tasks
|
65
|
+
# Directories containing the results from project-wide tasks
|
58
66
|
@@RESULT_DIRS = {
|
59
|
-
project_stats:
|
67
|
+
project_stats: '90.stats',
|
60
68
|
# Distances
|
61
|
-
haai_distances:
|
62
|
-
aai_distances:
|
63
|
-
ani_distances:
|
64
|
-
#ssu_distances:
|
69
|
+
haai_distances: '09.distances/01.haai',
|
70
|
+
aai_distances: '09.distances/02.aai',
|
71
|
+
ani_distances: '09.distances/03.ani',
|
72
|
+
# ssu_distances: '09.distances/04.ssu',
|
65
73
|
# Clade identification
|
66
|
-
clade_finding:
|
74
|
+
clade_finding: '10.clades/01.find',
|
67
75
|
# Clade analysis
|
68
|
-
subclades:
|
69
|
-
ogs:
|
70
|
-
#ess_phylogeny:
|
71
|
-
#core_phylogeny:
|
72
|
-
#clade_metadata:
|
76
|
+
subclades: '10.clades/02.ani',
|
77
|
+
ogs: '10.clades/03.ogs'
|
78
|
+
# ess_phylogeny: '10.clades/04.phylogeny/01.essential',
|
79
|
+
# core_phylogeny: '10.clades/04.phylogeny/02.core',
|
80
|
+
# clade_metadata: '10.clades/05.metadata'
|
73
81
|
}
|
74
82
|
|
75
83
|
##
|
76
|
-
# Supported types of projects
|
84
|
+
# Supported types of projects
|
77
85
|
@@KNOWN_TYPES = {
|
78
86
|
mixed: {
|
79
|
-
description:
|
80
|
-
single: true, multi: true
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
single: true, multi: false
|
85
|
-
|
86
|
-
|
87
|
+
description: 'Mixed collection of genomes, metagenomes, and viromes',
|
88
|
+
single: true, multi: true
|
89
|
+
},
|
90
|
+
genomes: {
|
91
|
+
description: 'Collection of genomes',
|
92
|
+
single: true, multi: false
|
93
|
+
},
|
94
|
+
clade: {
|
95
|
+
description: 'Collection of closely-related genomes (ANI >= 90%)',
|
96
|
+
single: true, multi: false
|
97
|
+
},
|
98
|
+
metagenomes: {
|
99
|
+
description: 'Collection of metagenomes and/or viromes',
|
100
|
+
single: false, multi: true
|
101
|
+
}
|
87
102
|
}
|
88
103
|
|
89
104
|
##
|
90
|
-
# Project-wide distance estimations
|
91
|
-
@@DISTANCE_TASKS = [
|
92
|
-
:haai_distances, :aai_distances, :ani_distances,
|
93
|
-
|
105
|
+
# Project-wide distance estimations
|
106
|
+
@@DISTANCE_TASKS = [
|
107
|
+
:project_stats, :haai_distances, :aai_distances, :ani_distances,
|
108
|
+
:clade_finding
|
109
|
+
]
|
110
|
+
|
94
111
|
##
|
95
|
-
# Project-wide tasks for :clade projects
|
112
|
+
# Project-wide tasks for :clade projects
|
96
113
|
@@INCLADE_TASKS = [:subclades, :ogs]
|
97
|
-
|
98
114
|
end
|
99
|
-
|
data/lib/miga/project/dataset.rb
CHANGED
@@ -4,11 +4,10 @@
|
|
4
4
|
##
|
5
5
|
# Helper module including specific functions handle datasets.
|
6
6
|
module MiGA::Project::Dataset
|
7
|
-
|
8
7
|
##
|
9
8
|
# Returns Array of MiGA::Dataset.
|
10
9
|
def datasets
|
11
|
-
metadata[:datasets].map{ |name| dataset(name) }
|
10
|
+
metadata[:datasets].map { |name| dataset(name) }
|
12
11
|
end
|
13
12
|
|
14
13
|
##
|
@@ -21,7 +20,7 @@ module MiGA::Project::Dataset
|
|
21
20
|
# Returns Hash of Strings => true. Similar to +dataset_names+ but as
|
22
21
|
# Hash for efficiency.
|
23
22
|
def dataset_names_hash
|
24
|
-
@dataset_names_hash ||= Hash[dataset_names.map{ |i| [i,true] }]
|
23
|
+
@dataset_names_hash ||= Hash[dataset_names.map { |i| [i, true] }]
|
25
24
|
end
|
26
25
|
|
27
26
|
##
|
@@ -29,6 +28,7 @@ module MiGA::Project::Dataset
|
|
29
28
|
def dataset(name)
|
30
29
|
name = name.miga_name
|
31
30
|
return nil unless MiGA::Dataset.exist?(self, name)
|
31
|
+
|
32
32
|
@datasets ||= {}
|
33
33
|
@datasets[name] ||= MiGA::Dataset.new(self, name)
|
34
34
|
@datasets[name]
|
@@ -62,6 +62,7 @@ module MiGA::Project::Dataset
|
|
62
62
|
def unlink_dataset(name)
|
63
63
|
d = dataset(name)
|
64
64
|
return nil if d.nil?
|
65
|
+
|
65
66
|
self.metadata[:datasets].delete(name)
|
66
67
|
save
|
67
68
|
pull_hook(:on_unlink_dataset, name)
|
@@ -71,28 +72,40 @@ module MiGA::Project::Dataset
|
|
71
72
|
##
|
72
73
|
# Import the dataset +ds+, a MiGA::Dataset, using +method+ which is any method
|
73
74
|
# supported by File#generic_transfer.
|
74
|
-
def import_dataset(ds, method
|
75
|
+
def import_dataset(ds, method = :hardlink)
|
75
76
|
raise "Impossible to import dataset, it already exists: #{ds.name}." if
|
76
77
|
MiGA::Dataset.exist?(self, ds.name)
|
78
|
+
|
77
79
|
# Import dataset results
|
78
80
|
ds.each_result do |task, result|
|
79
81
|
# import result files
|
80
82
|
result.each_file do |file|
|
81
|
-
File.generic_transfer(
|
82
|
-
|
83
|
+
File.generic_transfer(
|
84
|
+
File.join(result.dir, file),
|
85
|
+
File.join(path, 'data', MiGA::Dataset.RESULT_DIRS[task], file),
|
86
|
+
method
|
87
|
+
)
|
83
88
|
end
|
84
89
|
# import result metadata
|
85
90
|
%w(json start done).each do |suffix|
|
86
|
-
if File.exist?
|
87
|
-
File.generic_transfer(
|
88
|
-
"#{
|
89
|
-
|
91
|
+
if File.exist? File.join(result.dir, "#{ds.name}.#{suffix}")
|
92
|
+
File.generic_transfer(
|
93
|
+
File.join(result.dir, "#{ds.name}.#{suffix}"),
|
94
|
+
File.join(
|
95
|
+
path, 'data', MiGA::Dataset.RESULT_DIRS[task],
|
96
|
+
"#{ds.name}.#{suffix}"
|
97
|
+
),
|
98
|
+
method
|
99
|
+
)
|
90
100
|
end
|
91
101
|
end
|
92
102
|
end
|
93
103
|
# Import dataset metadata
|
94
|
-
File.generic_transfer(
|
95
|
-
|
104
|
+
File.generic_transfer(
|
105
|
+
File.join(ds.project.path, 'metadata', "#{ds.name}.json"),
|
106
|
+
File.join(self.path, 'metadata', "#{ds.name}.json"),
|
107
|
+
method
|
108
|
+
)
|
96
109
|
# Save dataset
|
97
110
|
self.add_dataset(ds.name)
|
98
111
|
end
|
@@ -104,11 +117,13 @@ module MiGA::Project::Dataset
|
|
104
117
|
MiGA::Dataset.RESULT_DIRS.values.each do |dir|
|
105
118
|
dir_p = "#{path}/data/#{dir}"
|
106
119
|
next unless Dir.exist? dir_p
|
120
|
+
|
107
121
|
Dir.entries(dir_p).each do |file|
|
108
122
|
next unless
|
109
123
|
file =~ %r{
|
110
124
|
\.(fa(a|sta|stqc?)?|fna|solexaqa|gff[23]?|done|ess)(\.gz)?$
|
111
125
|
}x
|
126
|
+
|
112
127
|
m = /([^\.]+)/.match(file)
|
113
128
|
datasets << m[1] unless m.nil? or m[1] == "miga-project"
|
114
129
|
end
|
@@ -146,5 +161,4 @@ module MiGA::Project::Dataset
|
|
146
161
|
def each_dataset_profile_advance(&blk)
|
147
162
|
each_dataset { |ds| blk.call(ds.profile_advance) }
|
148
163
|
end
|
149
|
-
|
150
164
|
end
|