bio-conduit 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5101f8f97fb43d2832036876835d7ecb7c6a690b
+  data.tar.gz: 94d96c406a17d7c5db2fe803c4b160b719b2e00b
+SHA512:
+  metadata.gz: 3257b904ef5687b8e15267b4f610b0ce7bc9603c2913df6050a995e4d5a300245c45ba1da80e45c1d36a47a58fb74f2363388a309176af2a3371f137f0a970b4
+  data.tar.gz: fdecf9dbe891ba147d5afa2fc25d418e10fdf406dc48fe68e64d69e5a125055dc8f694bb9207a67ac4b68ac6ad1025f51b18442ee9a37f2085d947547eba140a

data/README.md

@@ -0,0 +1,3 @@
+# README #
+
+This is a simple pipeline sunning library inspired by [bioruby-pipengine](https://github.com/fstrozzi/bioruby-pipengine)

data/bin/conduitrun

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+# bio-conduit
+#
+# Copyright (C) 2015
+#
+# author: Natapol Pornputtapong <natapol.por@gmail.com>
+#
+# Documentation: Natapol Pornputtapong (RDoc'd and embellished by William Webber)
+#
+
+# raise "Please, use ruby 1.9.0 or later." if RUBY_VERSION < "1.9.0"
+
+$:<< File.expand_path(File.join(File.dirname(File.dirname __FILE__),"lib"))
+
+require 'optparse'
+require 'bio-conduit'
+
+opts = {dry: false, queue_type: 'torque', pipeline: "pipeline.yml", sample_file: "samples.yml", queue_option: ""}
+OptionParser.new do |option|
+    option.banner = <<-EOS
+Test is an awesome program that does something very, very important.
+
+Usage:
+   #{$0} [options]
+where [options] are:
+EOS
+    option.on("-p", "--pipeline PIPELINE", "YAML file with pipeline information") {|v| opts[:pipeline] = v}
+    option.on("-f", "--sample_file SAMPLEFILE", "YAML file with samples information") {|v| opts[:sample_file] = v}
+    option.on("-l", "--samples SAMPLES", "List of sample names to run the pipeline") {|v| opts[:samples] = v.split(/,/)}
+    option.on("-s", "--steps STEPS", "List of steps to be executed") {|v| opts[:steps] = v.split(/,/)}
+    option.on("-d", "--dry", "YAML file with pipeline information") {|v| opts[:dry] = true}
+    option.on("-g", "--group GROUP", "Specify the group of samples to run the pipeline") {|v| opts[:group] = v.split(/,/)}
+    option.on("-o", "--output_dir OUTPUTDIR", "Output directory (override output directory in pipeline file)") {|v| opts[:output_dir] = v}
+    option.on("-q", "--queue_type QUEUETYPE", "Specify job scheduler") {|v| opts[:queue_type] = v}
+    option.on("-b", "--queue_option OPTIONS", "YAML file with pipeline information") {|v| opts[:queue_option] = v}
+end.parse!
+
+begin
+    sampleyml = Bio::Conduit::Samples.new(opts[:sample_file])
+rescue Errno::ENOENT
+    abort "ERR: No samples file - #{opts[:sample_file]}"
+end
+
+begin
+    procyml = Bio::Conduit::Process.new(opts[:pipeline])
+rescue Errno::ENOENT
+    abort "ERR: No pipeline file - #{opts[:pipeline]}"
+end
+
+samples = []
+if opts[:samples] != nil
+    opts[:samples].each do |sam|
+        samples.push(sampleyml[sam])
+    end
+elsif opts[:groups] != nil
+    sampleyml.groups[opts[:groups]].each do |sam|
+        samples.push(sam)
+    end
+else
+    samples = sampleyml.samples.values
+end
+
+samples.each do |sam|
+    template = Bio::Conduit::QUEUES[opts[:queue_type]][:template]
+    cmd = Bio::Conduit::QUEUES[opts[:queue_type]][:command].sub(/\<options\>/, opts[:queue_option])
+
+    runscript = procyml.run_steps(opts[:steps]).create_jobscript(sam, template)
+
+    if opts[:dry]
+        puts cmd
+        puts runscript
+    else
+        o, e, s = Open3.capture3(cmd, :stdin_data => runscript)
+        if !s.success?
+            raise Bio::Conduit::CommandError, "Process.run() of #{queuetype} job submission failed with #{e}"
+        end
+    end
+
+end

data/bio-conduit.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "bio/conduit/version"
+
+Gem::Specification.new do |s|
+    s.name          = 'bio-conduit'
+    s.version       = Bio::Conduit::VERSION
+    s.date          = Time.now.strftime("%Y-%m-%d")
+    s.platform      = Gem::Platform::RUBY
+
+    s.summary       = "A pipeline creator for bioruby"
+    s.description   = "A pipeline creator for bioruby"
+    s.authors       = ["Natapol Pornputtapong"]
+    s.email         = 'natapol.por@gmail.com'
+
+    s.homepage      = 'http://rubygems.org/gems/bio-conduit'
+    s.license       = 'GPL'
+
+
+    s.files         = `git ls-files`.split("\n")
+    s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+    s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+    s.require_paths = ["bin", "lib"]
+
+    #    if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("1.9.0")
+#        s.add_dependency "mongoid", "~> 3.1"
+#    else
+#        s.add_dependency "mongoid", "~> 4.0"
+#        s.add_dependency "mongoid-versioning", "~> 1.0"
+#    end
+    #s.add_dependency "trollop", "~> 2.0"
+    #s.add_dependency "bio", "~> 1.4"
+    #s.add_dependency "statsample", "~> 1.4"
+end

data/contributors.txt

@@ -0,0 +1 @@
+Natapol Pornputtapong

data/lib/bio-conduit.rb

@@ -0,0 +1,22 @@
+# bio-conduit
+#
+# Copyright (C) 2015
+#
+# author: Natapol Pornputtapong <natapol.por@gmail.com>
+#
+# Documentation: Natapol Pornputtapong (RDoc'd and embellished by William Webber)
+#
+
+# raise "Please, use ruby 1.9.0 or later." if RUBY_VERSION < "1.9.0"
+
+$:<< File.expand_path(File.join(File.dirname(File.dirname __FILE__),"lib"))
+
+require 'yaml'
+require 'erb'
+require 'fileutils'
+require 'open3'
+
+#require 'active_support/concern'
+#require 'active_support/core_ext/object/blank'
+
+require 'bio/conduit'

data/lib/bio/conduit.rb

@@ -0,0 +1,14 @@
+# bio-conduit
+#
+# Copyright (C) 2015
+#
+# author: Natapol Pornputtapong <natapol.por@gmail.com>
+#
+# Documentation: Natapol Pornputtapong (RDoc'd and embellished by William Webber)
+#
+require 'bio/conduit/version'
+require 'bio/conduit/exception'
+require 'bio/conduit/samples'
+require 'bio/conduit/process'
+require 'bio/conduit/template'
+require 'bio/conduit/queues'

data/lib/bio/conduit/exception.rb

@@ -0,0 +1,19 @@
+# bio-conduit
+#
+# Copyright (C) 2015
+#
+# author: Natapol Pornputtapong <natapol.por@gmail.com>
+#
+# Documentation: Natapol Pornputtapong (RDoc'd and embellished by William Webber)
+#
+module Bio
+    module Conduit
+        class FileNotFound < Exception
+
+        end
+
+        class CommandError < Exception
+
+        end
+    end
+end

data/lib/bio/conduit/process.rb

@@ -0,0 +1,147 @@
+# bio-conduit
+#
+# Copyright (C) 2015
+#
+# author: Natapol Pornputtapong <natapol.por@gmail.com>
+#
+# Documentation: Natapol Pornputtapong (RDoc'd and embellished by William Webber)
+#
+module Bio
+    module Conduit
+        class Step
+
+            attr_reader :name, :info, :resources
+
+            UNITCONV = {
+                '' => 1,
+                'b' => 1,
+                'k' => 1024,
+                'm' => 1048576,
+                'g' => 1073741824,
+                't' => 1099511627776
+            }
+
+            def initialize(name, hash, resources, addpath = true)
+                @name = name
+                @info = hash
+                @resources = resources
+
+                if addpath
+                    if @info['run'].is_a?(String)
+                        @info['run'] = ["mkdir -p <sample_output_path/>#{@name}", "cd <sample_output_path/>#{@name}", @info['run']]
+                    elsif @info['run'].is_a?(Array)
+                        @info['run'].unshift("mkdir -p <sample_output_path/>#{@name}", "cd <sample_output_path/>#{@name}")
+                    end
+                end
+            end
+
+            def dependence?
+                return @info.has_key?('pre')
+            end
+
+            def dependency
+                if self.dependence?
+                    return @info['pre']
+                else
+                    return nil
+                end
+            end
+
+            def +(other)
+                name = self.name + "_" + other.name
+                resources = self.resources.merge(other.resources)
+                info = self.info.merge(other.info) do |k, first, second|
+                    case k
+                    when 'mem', 'cpu', 'nodes'
+                        matched1 = /\A(\d+)([kmgtb]?)/.match(first)
+                        matched2 = /\A(\d+)([kmgtb]?)/.match(second)
+                        if (matched1[1].to_i * Bio::Conduit::Step::UNITCONV[matched1[2]]) > (matched2[1].to_i * Bio::Conduit::Step::UNITCONV[matched2[2]])
+                            first
+                        elsif (matched1[1].to_i * Bio::Conduit::Step::UNITCONV[matched1[2]]) < (matched2[1].to_i * Bio::Conduit::Step::UNITCONV[matched2[2]])
+                            second
+                        else
+                            first
+                        end
+                    else
+                        if first.is_a?(String) && second.is_a?(String)
+                            [first, second]
+                        elsif first.is_a?(Array) && second.is_a?(String)
+                            first + [second]
+                        elsif first.is_a?(String) && second.is_a?(Array)
+                            [first] + second
+                        else
+                            first + second
+                        end
+                    end
+                end
+                Bio::Conduit::Step.new(name, info, resources, false)
+            end
+
+            def create_jobscript(sample, template)
+                total_resources = @resources.merge(sample['resources'])
+                total_resources['sample_path'] = sample['path']
+                total_resources['sample'] = sample['name']
+                total_resources['sample_output_path'] = "#{total_resources['output'].chomp('/')}" + "#{sample.has_key?('group') ? "/#{sample['group']}" : ''}" + "/#{sample['name']}"
+                @jobname = sample['name'] + "_" + @name
+                @commands = @info['run'].is_a?(Array) ? @info['run'].join("\n") : @info['run'].clone
+                subslist = {}
+                @commands.scan(/(\<([\w\/]+)\>)/).each do |entry|
+                    entry[1].scan(/[^\/]+/).each do |res|
+                        entry[1].sub!(/#{res}/, total_resources[res])
+                    end
+                    @commands.sub!(/#{entry[0]}/, entry[1])
+                end
+                return ERB.new(template, nil, '-').result(binding)
+            end
+
+        end
+
+        class Process
+
+            attr_reader :steps
+
+            def initialize(filename)
+                yaml = YAML::load(File.open(filename))
+                @steps = {}
+                @name = yaml['pipeline']
+                @resources = yaml['resources']
+                @runninglist = []
+                yaml['steps'].each_pair do |name, detail|
+                    @resources[name] = "../#{name}"
+                    @steps[name] = Bio::Conduit::Step.new(name, detail, @resources)
+                end
+            end
+
+            def [](name)
+                @steps[name]
+            end
+
+            def add_running_step(stepname)
+                step = self[stepname]
+                self.add_running_step(step.dependency) if step.dependence?
+                if !@runninglist.include?(stepname)
+                    @runninglist.push(stepname)
+                end
+            end
+
+            def run_steps(steps)
+
+                @steps.keys.each do |name|
+                    self.add_running_step(name) if steps == nil || steps.include?(name)
+                end
+
+                combinestep = nil
+
+                @runninglist.each do |stepname|
+                    if combinestep
+                        combinestep = combinestep + self[stepname]
+                    else
+                        combinestep = self[stepname]
+                    end
+                end
+
+                return combinestep
+            end
+        end
+    end
+end

data/lib/bio/conduit/queues.rb

@@ -0,0 +1,27 @@
+# bio-conduit
+#
+# Copyright (C) 2015
+#
+# author: Natapol Pornputtapong <natapol.por@gmail.com>
+#
+# Documentation: Natapol Pornputtapong (RDoc'd and embellished by William Webber)
+#
+
+module Bio
+    module Conduit
+        QUEUES = {
+            "torque" => {
+                command: 'qsub <options> -',
+                template: Bio::Conduit::Template::TORQUE
+            },
+            "sge" => {
+                command: 'qsub <options> -',
+                template: Bio::Conduit::Template::SGE
+            },
+            "bash" => {
+                command: 'bash <options>',
+                template: Bio::Conduit::Template::SGE
+            }
+        }
+    end
+end

data/lib/bio/conduit/samples.rb

@@ -0,0 +1,61 @@
+# bio-conduit
+#
+# Copyright (C) 2015
+#
+# author: Natapol Pornputtapong <natapol.por@gmail.com>
+#
+# Documentation: Natapol Pornputtapong (RDoc'd and embellished by William Webber)
+#
+module Bio
+    module Conduit
+        class Samples
+
+            attr_reader :samples, :resources, :groups
+
+            def initialize(filename)
+                yaml = YAML::load(File.open(filename))
+
+                @samples = {}
+                @groups = {}
+
+                if yaml.has_key?("resources")
+                    @resources = yaml["resources"]
+                else
+                    @resources = {}
+                end
+
+                if yaml.has_key?("samples")
+                    yaml["samples"].each_pair do |l1k, l1v|
+                        if l1v.is_a?(Hash)
+                            l1v.each_pair do |k, v|
+                                @samples[k] = {}
+                                @samples[k]['name'] = k
+                                @samples[k]['path'] = v
+                                @samples[k]["resources"] = @resources
+                                @samples[k]["group"] = l1k
+                                @groups[l1k] ||= []
+                                @groups[l1k].push(@samples[k])
+                            end
+                        else
+                            @samples[l1k] = {}
+                            @samples[l1k]['name'] = l1k
+                            @samples[l1k]['path'] = l1v
+                            @samples[l1k]["resources"] = @resources
+                        end
+                    end
+                end
+
+            end
+
+            def [](name)
+                @samples[name]
+            end
+
+            def each(&block)
+                @samples.each do |sam|
+                    block.call(sam)
+                end
+            end
+        end
+    end
+end

data/lib/bio/conduit/template.rb

@@ -0,0 +1,11 @@
+# bio-conduit
+#
+# Copyright (C) 2015
+#
+# author: Natapol Pornputtapong <natapol.por@gmail.com>
+#
+# Documentation: Natapol Pornputtapong (RDoc'd and embellished by William Webber)
+#
+
+require 'bio/conduit/template/torque'
+require 'bio/conduit/template/sge'

data/lib/bio/conduit/template/sge.rb

@@ -0,0 +1,23 @@
+# bio-conduit
+#
+# Copyright (C) 2015
+#
+# author: Natapol Pornputtapong <natapol.por@gmail.com>
+#
+# Documentation: Natapol Pornputtapong (RDoc'd and embellished by William Webber)
+#
+module Bio
+    module Conduit
+        module Template
+            SGE = %q{#!/bin/bash
+#
+#$ -N <%= @jobname %>
+#$ -cwd
+#$ -S /bin/bash
+#
+<%= "#$ -pe multicores #{@info['cpu']}" if @info.has_key?('cpu') -%>
+
+<%= @commands -%>}
+        end
+    end
+end

data/lib/bio/conduit/template/torque.rb

@@ -0,0 +1,21 @@
+# bio-conduit
+#
+# Copyright (C) 2015
+#
+# author: Natapol Pornputtapong <natapol.por@gmail.com>
+#
+# Documentation: Natapol Pornputtapong (RDoc'd and embellished by William Webber)
+#
+module Bio
+    module Conduit
+        module Template
+            TORQUE = %q{#PBS -N <%= @jobname %>
+<% @resourceline = [] -%>
+<% @resourceline.push("nodes=#{@info.has_key?('nodes') ? @info['nodes'] : 1}#{@info.has_key?('cpu') ? ":ppn=#{@info['cpu']}" : ""}") if @info.has_key?('cpu') || @info.has_key?('nodes') -%>
+<% @resourceline.push("mem=#{@info['mem']}") if @info.has_key?('mem') -%>
+<%= "#PBS -l #{@resourceline.join(',')}" if !@resourceline.empty? %>
+
+<%= @commands -%>}
+        end
+    end
+end

data/lib/bio/conduit/version.rb

@@ -0,0 +1,14 @@
+# bio-conduit
+#
+# Copyright (C) 2015
+#
+# author: Natapol Pornputtapong <natapol.por@gmail.com>
+#
+# Documentation: Natapol Pornputtapong (RDoc'd and embellished by William Webber)
+#
+
+module Bio
+    module Conduit
+        VERSION = '1.0.0'
+    end
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: bio-conduit
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Natapol Pornputtapong
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-04 00:00:00.000000000 Z
+dependencies: []
+description: A pipeline creator for bioruby
+email: natapol.por@gmail.com
+executables:
+- conduitrun
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/conduitrun
+- bio-conduit.gemspec
+- contributors.txt
+- lib/bio-conduit.rb
+- lib/bio/conduit.rb
+- lib/bio/conduit/exception.rb
+- lib/bio/conduit/process.rb
+- lib/bio/conduit/queues.rb
+- lib/bio/conduit/samples.rb
+- lib/bio/conduit/template.rb
+- lib/bio/conduit/template/sge.rb
+- lib/bio/conduit/template/torque.rb
+- lib/bio/conduit/version.rb
+homepage: http://rubygems.org/gems/bio-conduit
+licenses:
+- GPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- bin
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: A pipeline creator for bioruby
+test_files: []
+has_rdoc: