rbbt-sources 3.0.25 → 3.0.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a527bda568cbbed6c8b2d1d97e9898fcd6b8b63c
-  data.tar.gz: 38dbe0d9ea4c5668f6368181c1b64a0790f2fb34
+  metadata.gz: 1eb41029cdf7f88ae9eb4d49b25dabfb90715670
+  data.tar.gz: a288b1baadc35a9a6d906432ab6dfc79be6cdb85
 SHA512:
-  metadata.gz: c8745c28b10e9b0312c06bbaf14ef42efa80467d3c64092a2440f426cd4139eae4151d93e9c471d66f6ecf2eeb26edd8d15c527369a14ac0908559f6c82f0a12
-  data.tar.gz: aecdcddd4dfd4e7474958aa6bb57ca8eaf06c346cd96e3c6515e8900224375641503976af8c60bb6949163ae53a79d6ede092512377218f0f3a028d556e02b5f
+  metadata.gz: 13d75ca6cb87be630047fb9ba5fc4d6f89173933c9070b4a3a4b6a89a84442be1eaac8856c6716b4b8eba626e81bfde89b86738e32b04b594c9bf6caf655c94c
+  data.tar.gz: f51c9a53315951d9d9333b796d2038bfc79a6438f25ada540cda5bfbd22619a3a73b98e2a340436c7942d43df2328220271576358932154785055e7473d03805

data/lib/rbbt/sources/synapse.rb

@@ -0,0 +1,218 @@
+module Synapse
+
+  PYTHON_LOGIN_TEMPLATE =<<-EOF
+import synapseclient
+syn = synapseclient.login()
+
+  EOF
+
+  PYTHON_FILE_TEMPLATE =<<-EOF
+f = synapseclient.File('[FILE]', parentId='[PARENTID]', name='[NAME]', contentType='[CONTENT-TYPE]')
+[ANNOTATIONS]
+
+  EOF
+  
+  PYTHON_SEND_TEMPLATE =<<-EOF
+f = syn.store(f, used= [USED], executed = [EXECUTED])
+print("syn: " + f.id)
+
+  EOF
+
+  PYTHON_DIR_TEMPLATE =<<-EOF
+f = synapseclient.Folder('[NAME]', parentId='[PARENTID]')
+f = syn.store(f)
+print("syn: " + f.id)
+
+  EOF
+
+  def self.python_login
+    PYTHON_LOGIN_TEMPLATE.dup
+  end
+
+  def self.python_create_dir(name, parentid)
+    name = File.basename(name)
+    template = PYTHON_DIR_TEMPLATE.dup
+
+    template.sub!('[PARENTID]', parentid)
+    template.sub!('[NAME]', name)
+
+    template
+  end
+
+  def self.python_upload_template(used = nil, executed = nil)
+    template = PYTHON_SEND_TEMPLATE.dup
+    if used
+      used_str = "#{used.inspect}"
+    else
+      used_str = "[]"
+    end
+
+    if executed
+      executed_str = "'#{executed}'"
+    else
+      executed_str = "''"
+    end
+
+    template.sub!('[USED]', used_str)
+    template.sub!('[EXECUTED]', executed_str)
+
+    template
+  end
+
+  def self.python_file_template(file, parentid, name = nil, content_type = nil, annotations = {})
+    template = PYTHON_FILE_TEMPLATE.dup
+
+    annotations = {} if annotations.nil?
+
+    name = annotations.delete :name if name.nil?
+    name = File.basename(file) if name.nil?
+
+    content_type = annotations.delete :content_type if content_type.nil?
+    content_type = begin
+                     MimeMagic.by_path(file) 
+                   rescue
+                     'text/plain'
+                   end if content_type.nil?
+
+    template.sub!('[FILE]', file)
+    template.sub!('[PARENTID]', parentid)
+    template.sub!('[NAME]', name)
+    template.sub!('[CONTENT-TYPE]', content_type)
+
+    if annotations
+      annotation_str = annotations.collect{|k,v| "f.#{k} = '#{v}'"} * "\n"
+      template.sub!('[ANNOTATIONS]', annotation_str)
+    else
+      template.sub!('[ANNOTATIONS]', '')
+    end
+
+    template
+  end
+
+  def self.upload_file(file, syn, name=nil, content_type=nil, annotations = {})
+    template = python_login + python_file_template(file, syn, name, content_type, annotations) + python_upload_template
+    syn = Misc.insist do
+      Log.warn "Uploading file: #{file}"
+      TmpFile.with_file(template) do |ftemplate|
+        `python '#{ ftemplate }'`.match(/syn: (syn\d+)/)[1]
+      end
+    end
+    syn
+  end
+
+  def self.upload_files(files, syn)
+    template = python_login 
+    order = []
+    files.each do |file,info|
+      order << file
+      name, content_type, annotations = info.values_at :name, :content_type, :annotations
+      template << python_file_template(file, syn, name, content_type, annotations) + python_upload_template
+    end
+
+    syns = Misc.insist do
+      Log.warn "Uploading files: #{Misc.fingerprint files}"
+      TmpFile.with_file(template) do |ftemplate|
+        res = `python '#{ ftemplate }'`
+        res.scan(/syn: syn\d+/).collect{|m| m.match(/(syn\d+)/)[0]}
+      end
+    end
+    Hash[*order.zip(syns).flatten]
+  end
+
+  def self.create_directory(dir, syn)
+    template = python_login + python_create_dir(dir, syn)
+    syn = Misc.insist do
+      Log.warn "Creating directory: #{dir}"
+      TmpFile.with_file(template) do |ftemplate|
+        `python '#{ ftemplate }'`.match(/syn: (syn\d+)/)[1]
+      end
+    end
+    syn
+  end
+
+  def self.create_directories(dirs, syn)
+    template = python_login 
+    order = []
+    dirs.each do |dir|
+      order << dir
+      template <<  python_create_dir(dir, syn)
+    end
+    syns = Misc.insist do
+      TmpFile.with_file(template) do |ftemplate|
+        Log.warn "Creating directories: #{Misc.fingerprint dirs}"
+        res = `python '#{ ftemplate }'`
+        res.scan(/syn: syn\d+/).collect{|m| m.match(/(syn\d+)/)[0]}
+      end
+    end
+    Hash[*order.zip(syns).flatten]
+  end
+
+  def self.directory_structure(directory)
+    structure = {}
+    Dir.glob(File.join(directory, '*')).each do |path|
+      if File.directory? path
+        structure[path] = directory_structure(path)
+      end
+    end
+    structure
+  end
+
+  def self.upload_dir(directory, syn, file_annotations = {})
+    Log.warn "Uploading #{ Log.color :blue, directory }"
+    files = Dir.glob(File.join(directory, '*')).reject{|f| File.directory? f}
+    file_info = {}
+    files.each{|file| file_info[file] = file_annotations[file] || {} } if Hash === file_annotations
+    files.each{|file| file_info[file] = file_annotations.call(file) || {} } if Proc === file_annotations
+    files.each{|file| file_info[file] = {} } if file_annotations.nil?
+
+    Log.warn "Uploading #{ Log.color :blue, directory } files: #{Misc.fingerprint files}"
+    upload_files(file_info, syn)
+
+    structure = directory_structure(directory)
+    directories = structure.keys
+    diretories_syns = create_directories(directories, syn)
+    Log.warn "Traversing #{ directory } subdirectories"
+    diretories_syns.each do |subdirectory,subsyn|
+      upload_dir(subdirectory, subsyn, file_annotations)
+    end
+  end
+end
+
+<<-'EOF'
+
+metadata = YAML.load(Open.open(metadata_file))
+IndiferentHash.setup metadata
+
+
+Log.info "Creating file #{file} in #{metadata[:parentid]}"
+syn = Misc.insist do
+  TmpFile.with_file(template) do |ftemplate|
+    `python '#{ ftemplate }'`.match(/syn: (syn\d+)/)[1]
+  end
+end
+
+Log.info "Created file #{file} in #{metadata[:parentid]}: #{ syn }"
+
+puts syn
+
+import synapseclient
+syn = synapseclient.login()
+
+[CONTENT-TYPE]
+
+f = synapseclient.File('[FILE]', parentId='[PARENTID]', name='[NAME]', contentType=contentType)
+
+#Lets set some annotations on the file
+#f.fileType = 'vcf'
+#f.pipeline = 'Sanger'
+#f.variant_type = 'indel'
+[ANNOTATIONS]
+
+#used = ['https://cghub.ucsc.edu/cghub/data/analysis/download/ba7ba416-a215-4bcf-b1b4-d6deaae0a645', 'https://cghub.ucsc.edu/cghub/data/analysis/download/7f733401-be7f-4fbe-b966-529f5574cbab']
+#executed = 'https://github.com/ICGC-TCGA-PanCancer/SeqWare-CGP-SomaticCore'
+[PROVENANCE]
+
+#Lets store the file to Synapse and set the provenance
+f = syn.store(f, used= used, executed = executed)
+print("syn: " + f.id)
+EOF

data/test/rbbt/sources/test_synapse.rb

@@ -0,0 +1,78 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt-util'
+require 'rbbt/sources/synapse'
+
+class TestSynapse < Test::Unit::TestCase
+  def sandbox_syn
+    "syn4231273"
+  end
+
+  def sftp_sandbox_syn
+    "syn4231379"
+  end
+
+
+  def _test_upload_file
+    TmpFile.with_file("test", true, :extension => 'tsv') do |filename|
+      ppp Synapse.upload_file(filename, sandbox_syn)
+    end
+  end
+
+  def _test_upload_files
+    TmpFile.with_file do |directory|
+      FileUtils.mkdir_p directory unless File.exists? directory
+      filename1 = File.join(directory, 'test1.txt')
+      Open.write(filename1, 'test1')
+      filename2 = File.join(directory, 'test2.txt')
+      Open.write(filename2, 'test2')
+
+      files = {filename1 => {}, filename2 => {}}
+
+      iii Synapse.upload_files(files, sandbox_syn)
+    end
+  end
+
+  def _test_create_directory
+    TmpFile.with_file() do |directory|
+      ppp Synapse.create_directory(File.basename(directory), sandbox_syn)
+    end
+  end
+
+  def _test_create_directories
+    ppp Synapse.create_directories([rand(100).to_s, rand(100).to_s], sandbox_syn)
+  end
+
+  def _test_upload_dir
+    TmpFile.with_file do |directory|
+      FileUtils.mkdir_p directory unless File.exists? directory
+      subdir1 = File.join(directory, 'subdir1')
+      FileUtils.mkdir_p subdir1 unless File.exists? subdir1
+      subdir2 = File.join(directory, 'subdir2')
+      FileUtils.mkdir_p subdir1 unless File.exists? subdir2
+
+      subdir3 = File.join(subdir2, 'subdir3')
+
+      Open.write(File.join(subdir1, 'file1.txt'), 'test1')
+      Open.write(File.join(subdir2, 'file2.txt'), 'test2')
+      Open.write(File.join(subdir3, 'file3.txt'), 'test3')
+
+      ppp `ls -R #{directory}`
+
+      annotation_proc = Proc.new do |path|
+        case path
+        when /file1/
+          {:content_type => "text/markdown"}
+        when /file2/
+          {:name => "Test file 2"}
+        when /file3/
+          nil
+        else
+          {}
+        end
+      end
+      newsyn = Synapse.create_directory('test_upload_dir.' + rand(100).to_s, sandbox_syn)
+      ppp Synapse.upload_dir(directory, newsyn, annotation_proc)
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-sources
 version: !ruby/object:Gem::Version
-  version: 3.0.25
+  version: 3.0.26
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-20 00:00:00.000000000 Z
+date: 2015-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -116,6 +116,7 @@ files:
 - lib/rbbt/sources/reactome.rb
 - lib/rbbt/sources/stitch.rb
 - lib/rbbt/sources/string.rb
+- lib/rbbt/sources/synapse.rb
 - lib/rbbt/sources/tfacts.rb
 - lib/rbbt/sources/uniprot.rb
 - lib/rbbt/sources/wgEncodeBroadHmm.rb
@@ -149,6 +150,7 @@ files:
 - test/rbbt/sources/test_pubmed.rb
 - test/rbbt/sources/test_stitch.rb
 - test/rbbt/sources/test_string.rb
+- test/rbbt/sources/test_synapse.rb
 - test/rbbt/sources/test_tfacts.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-sources
@@ -184,6 +186,7 @@ test_files:
 - test/rbbt/sources/test_entrez.rb
 - test/rbbt/sources/test_matador.rb
 - test/rbbt/sources/test_HPRD.rb
+- test/rbbt/sources/test_synapse.rb
 - test/rbbt/sources/test_go.rb
 - test/rbbt/sources/test_stitch.rb
 - test/rbbt/sources/test_organism.rb