rbbt-sources 3.0.25 → 3.0.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/sources/synapse.rb +218 -0
- data/test/rbbt/sources/test_synapse.rb +78 -0
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1eb41029cdf7f88ae9eb4d49b25dabfb90715670
|
4
|
+
data.tar.gz: a288b1baadc35a9a6d906432ab6dfc79be6cdb85
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 13d75ca6cb87be630047fb9ba5fc4d6f89173933c9070b4a3a4b6a89a84442be1eaac8856c6716b4b8eba626e81bfde89b86738e32b04b594c9bf6caf655c94c
|
7
|
+
data.tar.gz: f51c9a53315951d9d9333b796d2038bfc79a6438f25ada540cda5bfbd22619a3a73b98e2a340436c7942d43df2328220271576358932154785055e7473d03805
|
@@ -0,0 +1,218 @@
|
|
1
|
+
module Synapse
|
2
|
+
|
3
|
+
PYTHON_LOGIN_TEMPLATE =<<-EOF
|
4
|
+
import synapseclient
|
5
|
+
syn = synapseclient.login()
|
6
|
+
|
7
|
+
EOF
|
8
|
+
|
9
|
+
PYTHON_FILE_TEMPLATE =<<-EOF
|
10
|
+
f = synapseclient.File('[FILE]', parentId='[PARENTID]', name='[NAME]', contentType='[CONTENT-TYPE]')
|
11
|
+
[ANNOTATIONS]
|
12
|
+
|
13
|
+
EOF
|
14
|
+
|
15
|
+
PYTHON_SEND_TEMPLATE =<<-EOF
|
16
|
+
f = syn.store(f, used= [USED], executed = [EXECUTED])
|
17
|
+
print("syn: " + f.id)
|
18
|
+
|
19
|
+
EOF
|
20
|
+
|
21
|
+
PYTHON_DIR_TEMPLATE =<<-EOF
|
22
|
+
f = synapseclient.Folder('[NAME]', parentId='[PARENTID]')
|
23
|
+
f = syn.store(f)
|
24
|
+
print("syn: " + f.id)
|
25
|
+
|
26
|
+
EOF
|
27
|
+
|
28
|
+
def self.python_login
|
29
|
+
PYTHON_LOGIN_TEMPLATE.dup
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.python_create_dir(name, parentid)
|
33
|
+
name = File.basename(name)
|
34
|
+
template = PYTHON_DIR_TEMPLATE.dup
|
35
|
+
|
36
|
+
template.sub!('[PARENTID]', parentid)
|
37
|
+
template.sub!('[NAME]', name)
|
38
|
+
|
39
|
+
template
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.python_upload_template(used = nil, executed = nil)
|
43
|
+
template = PYTHON_SEND_TEMPLATE.dup
|
44
|
+
if used
|
45
|
+
used_str = "#{used.inspect}"
|
46
|
+
else
|
47
|
+
used_str = "[]"
|
48
|
+
end
|
49
|
+
|
50
|
+
if executed
|
51
|
+
executed_str = "'#{executed}'"
|
52
|
+
else
|
53
|
+
executed_str = "''"
|
54
|
+
end
|
55
|
+
|
56
|
+
template.sub!('[USED]', used_str)
|
57
|
+
template.sub!('[EXECUTED]', executed_str)
|
58
|
+
|
59
|
+
template
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.python_file_template(file, parentid, name = nil, content_type = nil, annotations = {})
|
63
|
+
template = PYTHON_FILE_TEMPLATE.dup
|
64
|
+
|
65
|
+
annotations = {} if annotations.nil?
|
66
|
+
|
67
|
+
name = annotations.delete :name if name.nil?
|
68
|
+
name = File.basename(file) if name.nil?
|
69
|
+
|
70
|
+
content_type = annotations.delete :content_type if content_type.nil?
|
71
|
+
content_type = begin
|
72
|
+
MimeMagic.by_path(file)
|
73
|
+
rescue
|
74
|
+
'text/plain'
|
75
|
+
end if content_type.nil?
|
76
|
+
|
77
|
+
template.sub!('[FILE]', file)
|
78
|
+
template.sub!('[PARENTID]', parentid)
|
79
|
+
template.sub!('[NAME]', name)
|
80
|
+
template.sub!('[CONTENT-TYPE]', content_type)
|
81
|
+
|
82
|
+
if annotations
|
83
|
+
annotation_str = annotations.collect{|k,v| "f.#{k} = '#{v}'"} * "\n"
|
84
|
+
template.sub!('[ANNOTATIONS]', annotation_str)
|
85
|
+
else
|
86
|
+
template.sub!('[ANNOTATIONS]', '')
|
87
|
+
end
|
88
|
+
|
89
|
+
template
|
90
|
+
end
|
91
|
+
|
92
|
+
def self.upload_file(file, syn, name=nil, content_type=nil, annotations = {})
|
93
|
+
template = python_login + python_file_template(file, syn, name, content_type, annotations) + python_upload_template
|
94
|
+
syn = Misc.insist do
|
95
|
+
Log.warn "Uploading file: #{file}"
|
96
|
+
TmpFile.with_file(template) do |ftemplate|
|
97
|
+
`python '#{ ftemplate }'`.match(/syn: (syn\d+)/)[1]
|
98
|
+
end
|
99
|
+
end
|
100
|
+
syn
|
101
|
+
end
|
102
|
+
|
103
|
+
def self.upload_files(files, syn)
|
104
|
+
template = python_login
|
105
|
+
order = []
|
106
|
+
files.each do |file,info|
|
107
|
+
order << file
|
108
|
+
name, content_type, annotations = info.values_at :name, :content_type, :annotations
|
109
|
+
template << python_file_template(file, syn, name, content_type, annotations) + python_upload_template
|
110
|
+
end
|
111
|
+
|
112
|
+
syns = Misc.insist do
|
113
|
+
Log.warn "Uploading files: #{Misc.fingerprint files}"
|
114
|
+
TmpFile.with_file(template) do |ftemplate|
|
115
|
+
res = `python '#{ ftemplate }'`
|
116
|
+
res.scan(/syn: syn\d+/).collect{|m| m.match(/(syn\d+)/)[0]}
|
117
|
+
end
|
118
|
+
end
|
119
|
+
Hash[*order.zip(syns).flatten]
|
120
|
+
end
|
121
|
+
|
122
|
+
def self.create_directory(dir, syn)
|
123
|
+
template = python_login + python_create_dir(dir, syn)
|
124
|
+
syn = Misc.insist do
|
125
|
+
Log.warn "Creating directory: #{dir}"
|
126
|
+
TmpFile.with_file(template) do |ftemplate|
|
127
|
+
`python '#{ ftemplate }'`.match(/syn: (syn\d+)/)[1]
|
128
|
+
end
|
129
|
+
end
|
130
|
+
syn
|
131
|
+
end
|
132
|
+
|
133
|
+
def self.create_directories(dirs, syn)
|
134
|
+
template = python_login
|
135
|
+
order = []
|
136
|
+
dirs.each do |dir|
|
137
|
+
order << dir
|
138
|
+
template << python_create_dir(dir, syn)
|
139
|
+
end
|
140
|
+
syns = Misc.insist do
|
141
|
+
TmpFile.with_file(template) do |ftemplate|
|
142
|
+
Log.warn "Creating directories: #{Misc.fingerprint dirs}"
|
143
|
+
res = `python '#{ ftemplate }'`
|
144
|
+
res.scan(/syn: syn\d+/).collect{|m| m.match(/(syn\d+)/)[0]}
|
145
|
+
end
|
146
|
+
end
|
147
|
+
Hash[*order.zip(syns).flatten]
|
148
|
+
end
|
149
|
+
|
150
|
+
def self.directory_structure(directory)
|
151
|
+
structure = {}
|
152
|
+
Dir.glob(File.join(directory, '*')).each do |path|
|
153
|
+
if File.directory? path
|
154
|
+
structure[path] = directory_structure(path)
|
155
|
+
end
|
156
|
+
end
|
157
|
+
structure
|
158
|
+
end
|
159
|
+
|
160
|
+
def self.upload_dir(directory, syn, file_annotations = {})
|
161
|
+
Log.warn "Uploading #{ Log.color :blue, directory }"
|
162
|
+
files = Dir.glob(File.join(directory, '*')).reject{|f| File.directory? f}
|
163
|
+
file_info = {}
|
164
|
+
files.each{|file| file_info[file] = file_annotations[file] || {} } if Hash === file_annotations
|
165
|
+
files.each{|file| file_info[file] = file_annotations.call(file) || {} } if Proc === file_annotations
|
166
|
+
files.each{|file| file_info[file] = {} } if file_annotations.nil?
|
167
|
+
|
168
|
+
Log.warn "Uploading #{ Log.color :blue, directory } files: #{Misc.fingerprint files}"
|
169
|
+
upload_files(file_info, syn)
|
170
|
+
|
171
|
+
structure = directory_structure(directory)
|
172
|
+
directories = structure.keys
|
173
|
+
diretories_syns = create_directories(directories, syn)
|
174
|
+
Log.warn "Traversing #{ directory } subdirectories"
|
175
|
+
diretories_syns.each do |subdirectory,subsyn|
|
176
|
+
upload_dir(subdirectory, subsyn, file_annotations)
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
<<-'EOF'
|
182
|
+
|
183
|
+
metadata = YAML.load(Open.open(metadata_file))
|
184
|
+
IndiferentHash.setup metadata
|
185
|
+
|
186
|
+
|
187
|
+
Log.info "Creating file #{file} in #{metadata[:parentid]}"
|
188
|
+
syn = Misc.insist do
|
189
|
+
TmpFile.with_file(template) do |ftemplate|
|
190
|
+
`python '#{ ftemplate }'`.match(/syn: (syn\d+)/)[1]
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
Log.info "Created file #{file} in #{metadata[:parentid]}: #{ syn }"
|
195
|
+
|
196
|
+
puts syn
|
197
|
+
|
198
|
+
import synapseclient
|
199
|
+
syn = synapseclient.login()
|
200
|
+
|
201
|
+
[CONTENT-TYPE]
|
202
|
+
|
203
|
+
f = synapseclient.File('[FILE]', parentId='[PARENTID]', name='[NAME]', contentType=contentType)
|
204
|
+
|
205
|
+
#Lets set some annotations on the file
|
206
|
+
#f.fileType = 'vcf'
|
207
|
+
#f.pipeline = 'Sanger'
|
208
|
+
#f.variant_type = 'indel'
|
209
|
+
[ANNOTATIONS]
|
210
|
+
|
211
|
+
#used = ['https://cghub.ucsc.edu/cghub/data/analysis/download/ba7ba416-a215-4bcf-b1b4-d6deaae0a645', 'https://cghub.ucsc.edu/cghub/data/analysis/download/7f733401-be7f-4fbe-b966-529f5574cbab']
|
212
|
+
#executed = 'https://github.com/ICGC-TCGA-PanCancer/SeqWare-CGP-SomaticCore'
|
213
|
+
[PROVENANCE]
|
214
|
+
|
215
|
+
#Lets store the file to Synapse and set the provenance
|
216
|
+
f = syn.store(f, used= used, executed = executed)
|
217
|
+
print("syn: " + f.id)
|
218
|
+
EOF
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
|
2
|
+
require 'rbbt-util'
|
3
|
+
require 'rbbt/sources/synapse'
|
4
|
+
|
5
|
+
class TestSynapse < Test::Unit::TestCase
|
6
|
+
def sandbox_syn
|
7
|
+
"syn4231273"
|
8
|
+
end
|
9
|
+
|
10
|
+
def sftp_sandbox_syn
|
11
|
+
"syn4231379"
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
def _test_upload_file
|
16
|
+
TmpFile.with_file("test", true, :extension => 'tsv') do |filename|
|
17
|
+
ppp Synapse.upload_file(filename, sandbox_syn)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def _test_upload_files
|
22
|
+
TmpFile.with_file do |directory|
|
23
|
+
FileUtils.mkdir_p directory unless File.exists? directory
|
24
|
+
filename1 = File.join(directory, 'test1.txt')
|
25
|
+
Open.write(filename1, 'test1')
|
26
|
+
filename2 = File.join(directory, 'test2.txt')
|
27
|
+
Open.write(filename2, 'test2')
|
28
|
+
|
29
|
+
files = {filename1 => {}, filename2 => {}}
|
30
|
+
|
31
|
+
iii Synapse.upload_files(files, sandbox_syn)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def _test_create_directory
|
36
|
+
TmpFile.with_file() do |directory|
|
37
|
+
ppp Synapse.create_directory(File.basename(directory), sandbox_syn)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def _test_create_directories
|
42
|
+
ppp Synapse.create_directories([rand(100).to_s, rand(100).to_s], sandbox_syn)
|
43
|
+
end
|
44
|
+
|
45
|
+
def _test_upload_dir
|
46
|
+
TmpFile.with_file do |directory|
|
47
|
+
FileUtils.mkdir_p directory unless File.exists? directory
|
48
|
+
subdir1 = File.join(directory, 'subdir1')
|
49
|
+
FileUtils.mkdir_p subdir1 unless File.exists? subdir1
|
50
|
+
subdir2 = File.join(directory, 'subdir2')
|
51
|
+
FileUtils.mkdir_p subdir1 unless File.exists? subdir2
|
52
|
+
|
53
|
+
subdir3 = File.join(subdir2, 'subdir3')
|
54
|
+
|
55
|
+
Open.write(File.join(subdir1, 'file1.txt'), 'test1')
|
56
|
+
Open.write(File.join(subdir2, 'file2.txt'), 'test2')
|
57
|
+
Open.write(File.join(subdir3, 'file3.txt'), 'test3')
|
58
|
+
|
59
|
+
ppp `ls -R #{directory}`
|
60
|
+
|
61
|
+
annotation_proc = Proc.new do |path|
|
62
|
+
case path
|
63
|
+
when /file1/
|
64
|
+
{:content_type => "text/markdown"}
|
65
|
+
when /file2/
|
66
|
+
{:name => "Test file 2"}
|
67
|
+
when /file3/
|
68
|
+
nil
|
69
|
+
else
|
70
|
+
{}
|
71
|
+
end
|
72
|
+
end
|
73
|
+
newsyn = Synapse.create_directory('test_upload_dir.' + rand(100).to_s, sandbox_syn)
|
74
|
+
ppp Synapse.upload_dir(directory, newsyn, annotation_proc)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-sources
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.
|
4
|
+
version: 3.0.26
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-05-
|
11
|
+
date: 2015-05-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rbbt-util
|
@@ -116,6 +116,7 @@ files:
|
|
116
116
|
- lib/rbbt/sources/reactome.rb
|
117
117
|
- lib/rbbt/sources/stitch.rb
|
118
118
|
- lib/rbbt/sources/string.rb
|
119
|
+
- lib/rbbt/sources/synapse.rb
|
119
120
|
- lib/rbbt/sources/tfacts.rb
|
120
121
|
- lib/rbbt/sources/uniprot.rb
|
121
122
|
- lib/rbbt/sources/wgEncodeBroadHmm.rb
|
@@ -149,6 +150,7 @@ files:
|
|
149
150
|
- test/rbbt/sources/test_pubmed.rb
|
150
151
|
- test/rbbt/sources/test_stitch.rb
|
151
152
|
- test/rbbt/sources/test_string.rb
|
153
|
+
- test/rbbt/sources/test_synapse.rb
|
152
154
|
- test/rbbt/sources/test_tfacts.rb
|
153
155
|
- test/test_helper.rb
|
154
156
|
homepage: http://github.com/mikisvaz/rbbt-sources
|
@@ -184,6 +186,7 @@ test_files:
|
|
184
186
|
- test/rbbt/sources/test_entrez.rb
|
185
187
|
- test/rbbt/sources/test_matador.rb
|
186
188
|
- test/rbbt/sources/test_HPRD.rb
|
189
|
+
- test/rbbt/sources/test_synapse.rb
|
187
190
|
- test/rbbt/sources/test_go.rb
|
188
191
|
- test/rbbt/sources/test_stitch.rb
|
189
192
|
- test/rbbt/sources/test_organism.rb
|