rbbt-sources 3.0.25 → 3.0.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a527bda568cbbed6c8b2d1d97e9898fcd6b8b63c
4
- data.tar.gz: 38dbe0d9ea4c5668f6368181c1b64a0790f2fb34
3
+ metadata.gz: 1eb41029cdf7f88ae9eb4d49b25dabfb90715670
4
+ data.tar.gz: a288b1baadc35a9a6d906432ab6dfc79be6cdb85
5
5
  SHA512:
6
- metadata.gz: c8745c28b10e9b0312c06bbaf14ef42efa80467d3c64092a2440f426cd4139eae4151d93e9c471d66f6ecf2eeb26edd8d15c527369a14ac0908559f6c82f0a12
7
- data.tar.gz: aecdcddd4dfd4e7474958aa6bb57ca8eaf06c346cd96e3c6515e8900224375641503976af8c60bb6949163ae53a79d6ede092512377218f0f3a028d556e02b5f
6
+ metadata.gz: 13d75ca6cb87be630047fb9ba5fc4d6f89173933c9070b4a3a4b6a89a84442be1eaac8856c6716b4b8eba626e81bfde89b86738e32b04b594c9bf6caf655c94c
7
+ data.tar.gz: f51c9a53315951d9d9333b796d2038bfc79a6438f25ada540cda5bfbd22619a3a73b98e2a340436c7942d43df2328220271576358932154785055e7473d03805
@@ -0,0 +1,218 @@
1
+ module Synapse
2
+
3
+ PYTHON_LOGIN_TEMPLATE =<<-EOF
4
+ import synapseclient
5
+ syn = synapseclient.login()
6
+
7
+ EOF
8
+
9
+ PYTHON_FILE_TEMPLATE =<<-EOF
10
+ f = synapseclient.File('[FILE]', parentId='[PARENTID]', name='[NAME]', contentType='[CONTENT-TYPE]')
11
+ [ANNOTATIONS]
12
+
13
+ EOF
14
+
15
+ PYTHON_SEND_TEMPLATE =<<-EOF
16
+ f = syn.store(f, used= [USED], executed = [EXECUTED])
17
+ print("syn: " + f.id)
18
+
19
+ EOF
20
+
21
+ PYTHON_DIR_TEMPLATE =<<-EOF
22
+ f = synapseclient.Folder('[NAME]', parentId='[PARENTID]')
23
+ f = syn.store(f)
24
+ print("syn: " + f.id)
25
+
26
+ EOF
27
+
28
+ def self.python_login
29
+ PYTHON_LOGIN_TEMPLATE.dup
30
+ end
31
+
32
+ def self.python_create_dir(name, parentid)
33
+ name = File.basename(name)
34
+ template = PYTHON_DIR_TEMPLATE.dup
35
+
36
+ template.sub!('[PARENTID]', parentid)
37
+ template.sub!('[NAME]', name)
38
+
39
+ template
40
+ end
41
+
42
+ def self.python_upload_template(used = nil, executed = nil)
43
+ template = PYTHON_SEND_TEMPLATE.dup
44
+ if used
45
+ used_str = "#{used.inspect}"
46
+ else
47
+ used_str = "[]"
48
+ end
49
+
50
+ if executed
51
+ executed_str = "'#{executed}'"
52
+ else
53
+ executed_str = "''"
54
+ end
55
+
56
+ template.sub!('[USED]', used_str)
57
+ template.sub!('[EXECUTED]', executed_str)
58
+
59
+ template
60
+ end
61
+
62
+ def self.python_file_template(file, parentid, name = nil, content_type = nil, annotations = {})
63
+ template = PYTHON_FILE_TEMPLATE.dup
64
+
65
+ annotations = {} if annotations.nil?
66
+
67
+ name = annotations.delete :name if name.nil?
68
+ name = File.basename(file) if name.nil?
69
+
70
+ content_type = annotations.delete :content_type if content_type.nil?
71
+ content_type = begin
72
+ MimeMagic.by_path(file)
73
+ rescue
74
+ 'text/plain'
75
+ end if content_type.nil?
76
+
77
+ template.sub!('[FILE]', file)
78
+ template.sub!('[PARENTID]', parentid)
79
+ template.sub!('[NAME]', name)
80
+ template.sub!('[CONTENT-TYPE]', content_type)
81
+
82
+ if annotations
83
+ annotation_str = annotations.collect{|k,v| "f.#{k} = '#{v}'"} * "\n"
84
+ template.sub!('[ANNOTATIONS]', annotation_str)
85
+ else
86
+ template.sub!('[ANNOTATIONS]', '')
87
+ end
88
+
89
+ template
90
+ end
91
+
92
+ def self.upload_file(file, syn, name=nil, content_type=nil, annotations = {})
93
+ template = python_login + python_file_template(file, syn, name, content_type, annotations) + python_upload_template
94
+ syn = Misc.insist do
95
+ Log.warn "Uploading file: #{file}"
96
+ TmpFile.with_file(template) do |ftemplate|
97
+ `python '#{ ftemplate }'`.match(/syn: (syn\d+)/)[1]
98
+ end
99
+ end
100
+ syn
101
+ end
102
+
103
+ def self.upload_files(files, syn)
104
+ template = python_login
105
+ order = []
106
+ files.each do |file,info|
107
+ order << file
108
+ name, content_type, annotations = info.values_at :name, :content_type, :annotations
109
+ template << python_file_template(file, syn, name, content_type, annotations) + python_upload_template
110
+ end
111
+
112
+ syns = Misc.insist do
113
+ Log.warn "Uploading files: #{Misc.fingerprint files}"
114
+ TmpFile.with_file(template) do |ftemplate|
115
+ res = `python '#{ ftemplate }'`
116
+ res.scan(/syn: syn\d+/).collect{|m| m.match(/(syn\d+)/)[0]}
117
+ end
118
+ end
119
+ Hash[*order.zip(syns).flatten]
120
+ end
121
+
122
+ def self.create_directory(dir, syn)
123
+ template = python_login + python_create_dir(dir, syn)
124
+ syn = Misc.insist do
125
+ Log.warn "Creating directory: #{dir}"
126
+ TmpFile.with_file(template) do |ftemplate|
127
+ `python '#{ ftemplate }'`.match(/syn: (syn\d+)/)[1]
128
+ end
129
+ end
130
+ syn
131
+ end
132
+
133
+ def self.create_directories(dirs, syn)
134
+ template = python_login
135
+ order = []
136
+ dirs.each do |dir|
137
+ order << dir
138
+ template << python_create_dir(dir, syn)
139
+ end
140
+ syns = Misc.insist do
141
+ TmpFile.with_file(template) do |ftemplate|
142
+ Log.warn "Creating directories: #{Misc.fingerprint dirs}"
143
+ res = `python '#{ ftemplate }'`
144
+ res.scan(/syn: syn\d+/).collect{|m| m.match(/(syn\d+)/)[0]}
145
+ end
146
+ end
147
+ Hash[*order.zip(syns).flatten]
148
+ end
149
+
150
+ def self.directory_structure(directory)
151
+ structure = {}
152
+ Dir.glob(File.join(directory, '*')).each do |path|
153
+ if File.directory? path
154
+ structure[path] = directory_structure(path)
155
+ end
156
+ end
157
+ structure
158
+ end
159
+
160
+ def self.upload_dir(directory, syn, file_annotations = {})
161
+ Log.warn "Uploading #{ Log.color :blue, directory }"
162
+ files = Dir.glob(File.join(directory, '*')).reject{|f| File.directory? f}
163
+ file_info = {}
164
+ files.each{|file| file_info[file] = file_annotations[file] || {} } if Hash === file_annotations
165
+ files.each{|file| file_info[file] = file_annotations.call(file) || {} } if Proc === file_annotations
166
+ files.each{|file| file_info[file] = {} } if file_annotations.nil?
167
+
168
+ Log.warn "Uploading #{ Log.color :blue, directory } files: #{Misc.fingerprint files}"
169
+ upload_files(file_info, syn)
170
+
171
+ structure = directory_structure(directory)
172
+ directories = structure.keys
173
+ diretories_syns = create_directories(directories, syn)
174
+ Log.warn "Traversing #{ directory } subdirectories"
175
+ diretories_syns.each do |subdirectory,subsyn|
176
+ upload_dir(subdirectory, subsyn, file_annotations)
177
+ end
178
+ end
179
+ end
180
+
181
+ <<-'EOF'
182
+
183
+ metadata = YAML.load(Open.open(metadata_file))
184
+ IndiferentHash.setup metadata
185
+
186
+
187
+ Log.info "Creating file #{file} in #{metadata[:parentid]}"
188
+ syn = Misc.insist do
189
+ TmpFile.with_file(template) do |ftemplate|
190
+ `python '#{ ftemplate }'`.match(/syn: (syn\d+)/)[1]
191
+ end
192
+ end
193
+
194
+ Log.info "Created file #{file} in #{metadata[:parentid]}: #{ syn }"
195
+
196
+ puts syn
197
+
198
+ import synapseclient
199
+ syn = synapseclient.login()
200
+
201
+ [CONTENT-TYPE]
202
+
203
+ f = synapseclient.File('[FILE]', parentId='[PARENTID]', name='[NAME]', contentType=contentType)
204
+
205
+ #Lets set some annotations on the file
206
+ #f.fileType = 'vcf'
207
+ #f.pipeline = 'Sanger'
208
+ #f.variant_type = 'indel'
209
+ [ANNOTATIONS]
210
+
211
+ #used = ['https://cghub.ucsc.edu/cghub/data/analysis/download/ba7ba416-a215-4bcf-b1b4-d6deaae0a645', 'https://cghub.ucsc.edu/cghub/data/analysis/download/7f733401-be7f-4fbe-b966-529f5574cbab']
212
+ #executed = 'https://github.com/ICGC-TCGA-PanCancer/SeqWare-CGP-SomaticCore'
213
+ [PROVENANCE]
214
+
215
+ #Lets store the file to Synapse and set the provenance
216
+ f = syn.store(f, used= used, executed = executed)
217
+ print("syn: " + f.id)
218
+ EOF
@@ -0,0 +1,78 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
+ require 'rbbt-util'
3
+ require 'rbbt/sources/synapse'
4
+
5
+ class TestSynapse < Test::Unit::TestCase
6
+ def sandbox_syn
7
+ "syn4231273"
8
+ end
9
+
10
+ def sftp_sandbox_syn
11
+ "syn4231379"
12
+ end
13
+
14
+
15
+ def _test_upload_file
16
+ TmpFile.with_file("test", true, :extension => 'tsv') do |filename|
17
+ ppp Synapse.upload_file(filename, sandbox_syn)
18
+ end
19
+ end
20
+
21
+ def _test_upload_files
22
+ TmpFile.with_file do |directory|
23
+ FileUtils.mkdir_p directory unless File.exists? directory
24
+ filename1 = File.join(directory, 'test1.txt')
25
+ Open.write(filename1, 'test1')
26
+ filename2 = File.join(directory, 'test2.txt')
27
+ Open.write(filename2, 'test2')
28
+
29
+ files = {filename1 => {}, filename2 => {}}
30
+
31
+ iii Synapse.upload_files(files, sandbox_syn)
32
+ end
33
+ end
34
+
35
+ def _test_create_directory
36
+ TmpFile.with_file() do |directory|
37
+ ppp Synapse.create_directory(File.basename(directory), sandbox_syn)
38
+ end
39
+ end
40
+
41
+ def _test_create_directories
42
+ ppp Synapse.create_directories([rand(100).to_s, rand(100).to_s], sandbox_syn)
43
+ end
44
+
45
+ def _test_upload_dir
46
+ TmpFile.with_file do |directory|
47
+ FileUtils.mkdir_p directory unless File.exists? directory
48
+ subdir1 = File.join(directory, 'subdir1')
49
+ FileUtils.mkdir_p subdir1 unless File.exists? subdir1
50
+ subdir2 = File.join(directory, 'subdir2')
51
+ FileUtils.mkdir_p subdir1 unless File.exists? subdir2
52
+
53
+ subdir3 = File.join(subdir2, 'subdir3')
54
+
55
+ Open.write(File.join(subdir1, 'file1.txt'), 'test1')
56
+ Open.write(File.join(subdir2, 'file2.txt'), 'test2')
57
+ Open.write(File.join(subdir3, 'file3.txt'), 'test3')
58
+
59
+ ppp `ls -R #{directory}`
60
+
61
+ annotation_proc = Proc.new do |path|
62
+ case path
63
+ when /file1/
64
+ {:content_type => "text/markdown"}
65
+ when /file2/
66
+ {:name => "Test file 2"}
67
+ when /file3/
68
+ nil
69
+ else
70
+ {}
71
+ end
72
+ end
73
+ newsyn = Synapse.create_directory('test_upload_dir.' + rand(100).to_s, sandbox_syn)
74
+ ppp Synapse.upload_dir(directory, newsyn, annotation_proc)
75
+ end
76
+ end
77
+ end
78
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-sources
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.25
4
+ version: 3.0.26
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-05-20 00:00:00.000000000 Z
11
+ date: 2015-05-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rbbt-util
@@ -116,6 +116,7 @@ files:
116
116
  - lib/rbbt/sources/reactome.rb
117
117
  - lib/rbbt/sources/stitch.rb
118
118
  - lib/rbbt/sources/string.rb
119
+ - lib/rbbt/sources/synapse.rb
119
120
  - lib/rbbt/sources/tfacts.rb
120
121
  - lib/rbbt/sources/uniprot.rb
121
122
  - lib/rbbt/sources/wgEncodeBroadHmm.rb
@@ -149,6 +150,7 @@ files:
149
150
  - test/rbbt/sources/test_pubmed.rb
150
151
  - test/rbbt/sources/test_stitch.rb
151
152
  - test/rbbt/sources/test_string.rb
153
+ - test/rbbt/sources/test_synapse.rb
152
154
  - test/rbbt/sources/test_tfacts.rb
153
155
  - test/test_helper.rb
154
156
  homepage: http://github.com/mikisvaz/rbbt-sources
@@ -184,6 +186,7 @@ test_files:
184
186
  - test/rbbt/sources/test_entrez.rb
185
187
  - test/rbbt/sources/test_matador.rb
186
188
  - test/rbbt/sources/test_HPRD.rb
189
+ - test/rbbt/sources/test_synapse.rb
187
190
  - test/rbbt/sources/test_go.rb
188
191
  - test/rbbt/sources/test_stitch.rb
189
192
  - test/rbbt/sources/test_organism.rb