cagnut_core 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e8f656425110b7d2bc1fbee6c89d45b97301e0b9
+  data.tar.gz: 056d7b62798bdc1254827276631111f1e2edbe49
+SHA512:
+  metadata.gz: c7fe94d621066a84390cbb05bb9f300333fea02b44693190c446958286cd045d50529f821e648225132402843aa8c4b42aef4a61d607aad86a6f38eb1ba0611e
+  data.tar.gz: 3a271af722e943e1a3af6e35efe6bc2d1f057ad968e27397a78496d4666efaff0c5de5a85ba18b09df42638980c0e5d0c80dd4389e8e775c97d0d8b37caf6b8c

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.3.1

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.12.5

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in cagnut_core.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Goldenio Technology
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+# CagnutCore
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'cagnut_core'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install cagnut_core
+
+## Usage
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/cagnut_core.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "cagnut_core"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/cagnut_core.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cagnut/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "cagnut_core"
+  spec.version       = CagnutCore::VERSION
+  spec.authors       = ['Shi-Gang Wang', 'Tse-Ching Ho']
+  spec.email         = ['seanwang@goldenio.com', 'tsechingho@goldenio.com']
+
+  spec.summary       = %q{Computational and Analytical Gear for Nucleic acid Utilitarian Techniques}
+  spec.description   = %q{Computational and Analytical Gear for Nucleic acid Utilitarian Techniques}
+  spec.homepage      = "https://github.com/CAGNUT/cagnut_core"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'activesupport', '> 5'
+  spec.add_dependency 'tilt'
+  spec.add_dependency 'thor'
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/exe/cagnut

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'thor'
+require 'optparse'
+require "cagnut"
+require "cagnut/cli/new_project"
+require "cagnut/cli/pipeline"
+
+method = ARGV.first
+case method
+when 'new', 'copy_params'
+  Cagnut::NewProject.start(ARGV)
+when 'pipeline'
+  Cagnut::Pipeline::Base.new.start(ARGV)
+else
+  puts <<-USAGE.strip_heredoc
+    cagnut new <folder_name>  create new project folder
+    cagnut pipeline           pipeline relative
+  USAGE
+end

data/exe/stat

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+require 'json'
+require 'time'
+
+dirs = ARGV
+
+if dirs.nil?
+  puts 'Please give *.std Working Directory'
+  exit
+end
+
+def file_name dir, file
+  %r{#{dir}\/(.+).[s][t][d]}.match(file)[1]
+end
+
+def check_resource line
+  case line
+  when 'Resource usage summary:'
+    @is_resource = true
+  when 'The output (if any) is above this job summary.'
+    @is_resource = false
+  end
+end
+
+def job_complete? line
+  @complete = true if line == 'Successfully completed.'
+end
+
+def cal_spend_time line
+  @started = fetch_time line, 'Started at ' if @started.nil?
+  @ended = fetch_time line, 'Results reported ' if @ended.nil?
+end
+
+def fetch_time line, marked
+  Time.parse(line.match(/#{marked}(.*+)/)[1]) if line.match(/#{marked}/)
+end
+
+def time_diff
+  Time.at(@ended - @started).utc.strftime '%H:%M:%S'
+end
+
+def check_content line
+  job_complete? line
+  check_resource line
+end
+
+def clasp_time
+  @complete ? time_diff : 'failed'
+end
+
+def reset_variable
+  @started = nil
+  @ended = nil
+  @complete = false
+  @is_resource = false
+end
+
+def parse_std file
+  resource_hash = {}
+  reset_variable
+  File.open(file).each_line do |line|
+    check_content line.chomp!
+    cal_spend_time line if @started.nil? || @ended.nil?
+    resource_hash['time'] = clasp_time
+    next unless line.length > 0 && @complete && @is_resource
+    data = line.gsub(/\s\s+/, '').split(':')
+    resource_hash[data[0]] = data[1] unless data[1].nil?
+  end
+  resource_hash
+end
+
+dirs.each do |dir|
+  stat = []
+  Dir.glob("#{dir}/*.std").each do |file|
+    resource_hash = parse_std file
+    stat << [file_name(dir, file), resource_hash]
+  end
+  puts stat.to_json.inspect
+end

data/lib/cagnut.rb

@@ -0,0 +1,36 @@
+require 'tilt'
+require 'active_support'
+require 'active_support/core_ext'
+require 'fileutils'
+require 'pathname'
+require 'cagnut/version'
+require 'cagnut/configuration'
+require 'cagnut/job_manage'
+
+Tilt.register_lazy :StringTemplate, 'tilt/string', 'sh'
+
+module Cagnut
+  autoload :Base, 'cagnut/base'
+
+  class << self
+    attr_writer :environment
+
+    def root
+      ::Pathname.new File.expand_path '../..', __FILE__
+    end
+
+    # Job names can contain up to 4094 characters.
+    def prefix_name
+      "CAGNUT_#{Time.now.strftime('%Y%m%d%H%M%S')}"
+    end
+
+    def environment
+      @environment ||= 'development'
+    end
+
+    def load_config config_name, config_options
+      Cagnut::Configuration.config = Cagnut::Configuration.load_config config_name, config_options
+    end
+
+  end
+end

data/lib/cagnut/cli/new_project.rb

@@ -0,0 +1,97 @@
+module Cagnut
+  class NewProject < Thor
+
+    include Thor::Actions
+    source_root Cagnut.root.join('templates')
+
+    desc 'new NAME project', 'new NAME project'
+    def new *args
+      if args.size >= 1
+        if !args.first.start_with? '-'
+          name = args.shift
+        elsif !args.last.start_with? '-'
+          name = args.pop
+        else
+          puts "please use `cagnut new <name>`\n"
+          exit(1)
+        end
+      else
+        puts "please use `cagnut new <name>`\n"
+        exit(1)
+      end
+      new_project name, project_opts
+    end
+
+    private
+
+    def new_project name, options
+      empty_directory name
+      copy_file 'Gemfile', "#{name}/Gemfile"
+      inside name, verbose: true do
+        create_file '.ruby-version', '2.3.1'
+        append_to_file 'Gemfile', "gem 'cagnut_cluster'\n" if options[:cluster]
+        append_pipeline_gems_to_gemfile options[:pipelines]
+        bundle 'install'
+      end
+      copy_file 'system.yml', "#{name}/system.yml"
+      load_bundle_env name
+      after_new_project name
+      generate_pipeline_tools_config name, options[:pipelines], options[:cluster]
+    end
+
+    def add_queue_setting name, pipeline
+    end
+
+    def load_bundle_env name
+      ENV['BUNDLE_GEMFILE'] ||= File.expand_path("#{name}/Gemfile", Dir.pwd)
+      require 'bundler/setup'
+      Bundler.require(:default)
+    end
+
+    def append_pipeline_gems_to_gemfile pipelines
+      return if pipelines.blank?
+      pipelines.each do |pipeline_name|
+        append_to_file 'Gemfile', "gem 'cagnut_pipeline_#{pipeline_name}'\n"
+      end
+    end
+
+    def generate_pipeline_tools_config name, pipelines, cluster=nil
+      return if pipelines.blank?
+      pipelines.each do |pipeline_name|
+        send "copy_#{pipeline_name}_tools_config", name
+        add_queue_setting name, pipeline_name if cluster
+      end
+    end
+
+    def after_new_project name
+    end
+
+    def project_opts options = {}
+      OptionParser.new do |opts|
+        opts.banner = 'Usage: example.rb [options]'
+        opts.on('-c', '--cluster', 'Cluster') do
+          options[:cluster] = true
+        end
+        opts.on('-p', '--pipelines draw', Array, 'Pipelines') do |p|
+          options[:pipelines] = p
+        end
+      end.parse!
+      return options
+    end
+
+    def bundle command
+      say_status :run, "bundle #{command}"
+      _bundle = Gem.bin_path('bundler', 'bundle')
+      require 'bundler'
+      Bundler.with_clean_env do
+        full_command = %Q["#{Gem.ruby}" "#{_bundle}" #{command}]
+        if options[:quiet]
+          system(full_command, out: File::NULL)
+        else
+          system(full_command)
+        end
+      end
+    end
+
+  end
+end

data/lib/cagnut/cli/pipeline.rb

@@ -0,0 +1,97 @@
+module Cagnut
+  module Pipeline
+    class Base
+
+      class << self
+        attr_reader :pipelines
+
+        def register klass
+          @pipelines ||= []
+          @pipelines << klass
+        end
+      end
+
+      def start *args
+        ENV['BUNDLE_GEMFILE'] ||= File.expand_path('./Gemfile', Dir.pwd)
+        require 'bundler/setup'
+        Bundler.require(:default)
+
+        options = pepeline_opts
+        run_filter options[:run] if options[:run]
+        config_name = get_config_name options[:run]
+        config = Cagnut.load_config config_name, options
+        config['dodebug'] = options[:debug]
+        config['samples'].each do |sample|
+          config['sample'] = sample
+          run_pipeline options[:run] if options[:run]
+          execute_command config, sample unless options[:debug]
+        end
+      end
+
+      def get_config_name pipeline_names
+        abort 'Did not assign pipeline to run' if pipeline_names.blank?
+        selected = self.class.pipelines.find do |p|
+          if p.try(:pipeline_names)
+            !( pipeline_names & p.pipeline_names ).empty?
+          elsif p.try(:pipeline_name)
+            pipeline_names.include? p.pipeline_name
+          else
+            puts "Can not find tools yml"
+            exit
+          end
+        end
+        selected.config_name
+      end
+
+      def run_filter names
+        puts 'Pipeline Conflict!'
+        exit
+      end
+
+      def run_pipeline pipelines, job_name = '', filename = ''
+        pipelines.sort.each do |pipeline|
+          job_name, filename = send "pipeline_#{pipeline}", { job_name: job_name, filename: filename }
+        end
+      end
+
+      def execute_command config, sample
+        job = fork do
+          exec "#{sample['jobs']}/submit_command_#{sample['name']}.jobs"
+        end
+        Process.detach(job)
+      end
+
+      def pepeline_opts options = {}
+        OptionParser.new do |opts|
+          opts.banner = 'Usage: example.rb [options]'
+          opts.on('-d', '--debug', 'Dodebug') do
+            options[:debug] = true
+          end
+          opts.on('-c', '--config yaml', 'Cagnut Config YAML') do |c|
+            options[:config] = c
+          end
+          opts.on('-n', '--no_check_tools', 'Not Check Tools') do
+            options[:no_check] = true
+          end
+          opts.on('-r', '--run draw1,draw2,draw3 or xyz', Array, 'run: draw1,draw2,draw3 or xyz') do |r|
+            options[:run] = r
+          end
+          opts.on('-p', '--parameter yaml', 'Cagnut Parameter Config YAML') do |p|
+            options[:params] = p
+          end
+          opts.on('-l', '--list', 'Pipeline List') do
+            list = self.class.pipelines.map do |pipeline|
+              pipeline.try(:pipeline_names) || pipeline.try(:pipeline_name)
+            end
+            puts "\s\s#{list.join("\n\s\s")}"
+          end
+          opts.on('-h', '--help', 'Help') do
+            puts "Help"
+          end
+        end.parse!
+        return options
+      end
+
+    end
+  end
+end

data/lib/cagnut/configuration.rb

@@ -0,0 +1,88 @@
+require 'yaml'
+require 'forwardable'
+require 'cagnut/configuration/base'
+require 'cagnut/configuration/checks/tools'
+require 'cagnut/configuration/checks/datasets'
+
+module Cagnut
+  module Configuration
+    mattr_writer :base, :toolbox
+    mattr_accessor :config, :params
+
+    class << self
+      def configure
+        yield self
+      end
+
+      def params
+        @params
+      end
+
+      def base
+        @base ||= begin
+          Cagnut::Configuration::Base.load(@config)
+          Cagnut::Configuration::Base.instance
+        end
+      end
+
+      def load_config config_name, options
+        @config ||= check_and_load_yml fetch_system_config_path(options[:config])
+        @params ||= check_and_load_yml fetch_tools_config_path(config_name, options[:params])
+        config_check config_name, options[:not_check]
+      end
+
+      private
+
+      def check_and_load_yml yml_path
+        check_path yml_path
+        puts "Using #{yml_path}"
+        YAML.load_file yml_path
+      end
+
+      def check_path file
+        return file if File.exist?(file)
+        puts "No such File in: #{file}"
+        exit
+      end
+
+      def fetch_system_config_path cfg_path=nil
+        if !cfg_path.blank?
+          cfg_path
+        elsif Dir.entries(Dir.pwd).include? "system.yml"
+          File.join(Dir.pwd, 'system.yml')
+        else
+          puts "Not Found system.yml in #{Dir.pwd}"
+          exit
+        end
+      end
+
+      def fetch_tools_config_path config_name, config_path=nil
+        if !config_path.blank?
+          config_path
+        elsif Dir.entries(Dir.pwd).include? "#{config_name}_tools.yml"
+          File.join(Dir.pwd, "#{config_name}_tools.yml")
+        else
+          puts "Not Found #{config_name}_tools.yml in #{Dir.pwd}"
+          exit
+        end
+      end
+
+      def config_check config_name, not_check=false
+        tools_check unless not_check
+        @config['pipeline_name'] = config_name
+        @config = check_datasets config_name
+      end
+
+      def tools_check
+        @tools = Cagnut::Configuration::Checks::Tools.new @config
+        @tools.check
+      end
+
+      def check_datasets config_name
+        @datasets = Cagnut::Configuration::Checks::Datasets.new @config
+        @datasets.check config_name
+      end
+
+    end
+  end
+end

data/lib/cagnut/configuration/base.rb

@@ -0,0 +1,44 @@
+require 'singleton'
+
+module Cagnut
+  module Configuration
+    class Base
+      include Singleton
+      attr_accessor :ref_fasta, :data_type, :snpdb, :sample_name, :jobs_dir, :java_path,
+                    :prefix_name, :target_flanks_file, :dbsnp_ref_indels, :cluster,
+                    :target, :magic28, :dodebug, :seqs_path, :pipeline_name
+
+      class << self
+        def load config
+          instance.load config
+        end
+      end
+
+      def load config
+        @config = config
+        attributes.each do |name, value|
+          send "#{name}=", value if respond_to? "#{name}="
+        end
+      end
+
+      def attributes
+        {
+          prefix_name: @config['prefix_name'],
+          sample_name: @config['sample']['name'],
+          dodebug: @config['dodebug'],
+          java_path: @config['tools']['java'],
+          ref_fasta: @config['refs']['ref_fasta'],
+          snpdb: @config['refs']['dbsnp']['ref'],
+          dbsnp_ref_indels: @config['refs']['dbsnp']['indels'],
+          target: @config['refs']['targets_file'],
+          target_flanks_file: @config['refs']['target_flanks_file'],
+          magic28: '1f8b08040000000000ff0600424302001b0003000000000000000000',
+          seqs_path: @config['sample']['seqs_path'],
+          data_type: @config['info']['data_type'],
+          jobs_dir: @config['sample']['jobs'],
+          cluster: @config['cluster']
+        }
+      end
+    end
+  end
+end

data/lib/cagnut/configuration/checks/datasets.rb

@@ -0,0 +1,149 @@
+module Cagnut
+  module Configuration
+    module Checks
+
+      class Datasets
+        attr_accessor :config
+
+        def initialize config
+          @config = config
+        end
+
+        def check config_name
+          @config['prefix_name'] = "#{Cagnut.prefix_name}_#{config_name}"
+          analysis_folder = create_analysis_folder config
+          @config['samples'].each_with_index do |sample, index|
+            setup_requirements sample, index, analysis_folder
+          end
+          @config
+        end
+
+        def create_analysis_folder config
+          output_data_dir = dir_rm_slash @config['cagnut']['output_data_dir']
+          analysis_folder = "#{output_data_dir}/#{config['prefix_name']}"
+          FileUtils.mkdir_p "#{output_data_dir}/#{config['prefix_name']}"
+          analysis_folder
+        end
+
+        def setup_requirements sample, index, analysis_folder
+          puts "Dataset : #{sample['path']}"
+          dir = "#{analysis_folder}/#{sample['name']}"
+          FileUtils.mkdir_p dir unless Dir.exist?(dir)
+          @config['samples'][index]['path'] = dir_rm_slash sample['path']
+          FileUtils.mkdir_p "#{analysis_folder}/#{sample['name']}/jobs"
+          @config['samples'][index]['jobs'] = "#{analysis_folder}/#{sample['name']}/jobs"
+          FileUtils.mkdir_p "#{analysis_folder}/#{sample['name']}/tmp"
+          @config['samples'][index]['tmp'] = "#{analysis_folder}/#{sample['name']}/tmp"
+          # mysql_insert if options[:mysql]
+          make_required_folders @config, sample, index, dir
+          check_pu index
+        end
+
+        def make_required_folders config, sample, index, analysis_folder
+        end
+
+        def ln_seq_files_to_folder sample, qseq_path, fastq_path
+          ln_seq_files sample, qseq_path, fastq_path
+          check_datatype qseq_path, fastq_path
+          check_ln_file sample, qseq_path, fastq_path
+        end
+
+        def check_ln_file sample, qseq_path, fastq_path
+          fastq = Dir.glob("#{fastq_path}/*.fastq*")
+          qseq = Dir.glob("#{qseq_path}/*")
+          return unless (fastq + qseq).empty?
+          abort "Not found #{sample['name']} files in fastq and qseq"
+        end
+
+        def ln_seq_files sample, seq_txt, fastq_file
+          dir_present? sample['path']
+          ln_seq_txt_file sample, seq_txt
+          ln_fastq_file sample, fetch_flist(sample['path']), fastq_file
+        end
+
+        def dir_present? dataset
+          return if Dir.exist?(dataset)
+          puts "Error: Missing data directory #{@config['datasets']}"
+          exit
+        end
+
+        def dir_rm_slash dir
+          dir.gsub %r{/\z}, ''
+        end
+
+        def ln_seq_txt_file sample, qseq_dir
+          files =
+            Dir.glob("#{sample['path']}/*_sequence.txt*") + Dir.glob("#{sample['path']}/*_qseq.txt*")
+          files.each do |f|
+            `ln -s #{f} #{qseq_dir} 2>/dev/null` if f.match sample['name']
+          end
+        end
+
+        def fetch_flist dir
+          flist = Dir.glob("#{dir}/*.fastq*")
+          return flist unless flist.empty?
+          abort "No fastq found in #{dir}"
+        end
+
+        def ln_fastq_file sample, flist, fastq_dir
+          if %w(ONEFASTQ ONEFASTQSE).include? @config['info']['data_type']
+            files_to_much? flist
+            file_type = link_name flist, sample['name']
+            seq_file = "#{fastq_dir}/#{file_type}"
+            `ln -s #{flist[0]} #{seq_file} 2>/dev/null` if flist[0].match sample['name']
+          else
+            flist.each do |f|
+              next unless f.match sample['name']
+              `ln -s #{f} #{fastq_dir} 2>/dev/null`
+            end
+          end
+        end
+
+        def link_name flist, sample_name
+          if flist[0].match '.gz'
+            "#{sample_name}_sequence.txt.gz"
+          else
+            "#{sample_name}_sequence.txt"
+          end
+        end
+
+        def files_to_much? flist
+          return unless flist.size > 1
+          puts %(
+            DATA_TYPE = #{@config['info']['data_type']} but more than one fastq found.
+            Only the first would be processed.
+            #{flist.inspect}
+          )
+        end
+
+        def check_pu index
+          @config['samples'][index]['pu'] ||= 'NA'
+        end
+
+        def check_datatype qseq_dir, fastq_dir
+          @config['samples'].each_with_index do |sample, index|
+            case @config['info']['data_type']
+            when 'TILESQSEQ'
+              file = "#{qseq_dir}/*.txt*"
+              pattern = '.*s_\d+_1_(\d+).*'
+              file_end = '.fastq'
+            when 'TILESFASTQ'
+              file = "#{fastq_dir}/*.fastq*"
+              pattern = '(.*_R1_.*).fastq.*+'
+              file_end = '.fastq'
+            end
+            @config['samples'][index]['seqs_path']= fetch_seqs Dir[file], file_end, pattern
+          end
+          @config
+        end
+
+        def fetch_seqs files_path, file_end, pattern
+          files_path.map do |file|
+            return file if File.basename(file, file_end).match(/#{pattern}/)
+          end.flatten.compact
+        end
+      end
+
+    end
+  end
+end

data/lib/cagnut/configuration/checks/tools.rb

@@ -0,0 +1,76 @@
+module Cagnut
+  module Configuration
+    module Checks
+      class Tools
+        attr_accessor :config, :check_completed, :java
+
+        def initialize config
+          @config = config
+        end
+
+        def check
+          @check_completed = true
+          check_each_tool
+          result = @check_completed ? 'Completed!' : 'Failed!'
+          puts "Check Tools: #{result}"
+          exit unless @check_completed
+        end
+
+        def check_each_tool
+          tools = @config['tools']
+          refs = @config['refs']
+          puts 'Start Checking...'
+          check_queueing_system @config['queueing_system']
+          check_ref_fasta refs['ref_fasta']
+          check_java tools['java']
+          check_r tools['R']
+          check_tool tools, refs
+        end
+
+        def check_tool tools_path, refs=nil
+        end
+
+        def check_tool_ver tool
+          ver = yield if block_given?
+          @check_completed = false if ver.nil?
+          ver = ver.nil? ? 'Not Found' : ver.chomp!
+          puts "Using #{tool} (#{ver})"
+        end
+
+        def check_java path
+          failed = check_tool_ver 'Java' do
+            `#{path} -version 2>&1| grep version | cut -f3 -d ' '` if path
+          end
+          @java = path unless failed
+        end
+
+        def check_r path
+          check_tool_ver 'R' do
+            `#{path} --version 2>&1 |grep ' version '| cut -f3 -d ' '` if path
+          end
+          check_r_libs path if path
+        end
+
+        def check_r_libs r_path
+          %w(gplots ggplot2 reshape gsalib).each do |lib|
+            check_tool_ver "R library: #{lib}" do
+              `#{r_path}script -e 'packageVersion("#{lib}")' | cut -f2 -d ' '`
+            end
+          end
+        end
+
+        def check_queueing_system queueing_system
+          system = queueing_system.nil? ? 'Local' : queueing_system['system']
+          puts "Using Queueing System: #{system}"
+        end
+
+        def check_ref_fasta ref_path
+          puts 'Checking Reference Files...'
+          return if File.exist?(ref_path)
+          puts "\tReference not founded in #{ref_path}"
+          @check_completed = false
+        end
+      end
+    end
+  end
+end

data/lib/cagnut/job_manage.rb

@@ -0,0 +1,63 @@
+require 'singleton'
+
+module Cagnut
+  class JobManage
+    include Singleton
+    extend Forwardable
+    def_delegators :'Cagnut::Configuration.base', :sample_name, :jobs_dir, :cluster
+
+    class << self
+      def submit job_script, job_name, opts
+        instance.submit job_script, job_name, opts
+      end
+
+      def run_local
+        instance.run_local
+      end
+    end
+
+    def submit job_script, job_name, opts
+      command = full_command job_script, job_name, opts
+      export_command command
+      puts command
+    end
+
+    def full_command job_script, job_name, opts
+      super if defined?(super)
+      return unless local_run?
+      command = local job_script
+    end
+
+    def local_run?
+      cluster.blank? || cluster['system'] == 'Local'
+    end
+
+    def export_command command
+      file = File.join jobs_dir, "submit_command_#{sample_name}.jobs"
+      File.open(file, 'a') do |f|
+        f.puts <<-BASH.strip_heredoc
+          #{command}
+          #{wait_local}
+        BASH
+      end
+      File.chmod(0700, file)
+    end
+
+    def local job_script
+      %(nohup #{jobs_dir}/#{job_script}.sh \
+      > #{jobs_dir}/#{job_script}.std \
+      2>#{jobs_dir}/#{job_script}.err &)
+    end
+
+    def run_local
+      return unless local_run?
+      %(& echo $! >> #{jobs_dir}/submit_job_#{sample_name}.ids
+        wait $!)
+    end
+
+    def wait_local
+      return unless local_run?
+      'wait $!'
+    end
+  end
+end

data/lib/cagnut/version.rb

@@ -0,0 +1,3 @@
+module CagnutCore
+  VERSION = "0.3.0"
+end

data/lib/cagnut_core.rb

@@ -0,0 +1 @@
+require 'cagnut'

data/templates/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem 'cagnut_core'

data/templates/system.yml

@@ -0,0 +1,48 @@
+###### ACTGUN Configuration
+cagnut:
+  #project_dir is used as the data output location
+  output_data_dir: '/home/u00wsg00/work3/data/'
+
+tools:
+  bwa: '/home/u00wsg00/prg/bwa/bwa-0.7.12/bwa'
+  samtools: '/home/u00wsg00/prg/samtools/samtools/bin/samtools'
+  gatk: '/home/u00wsg00/prg/gatk/3.4.0/GenomeAnalysisTK.jar'
+  picard: '/home/u00wsg00/prg/picard/picard/dist/picard.jar'
+  java: '/pkg/java/jre/bin/java'
+  R: '/home/u00wsg00/prg/R/R-3.2.1/bin/R'
+  snpeff: '/home/u00wsg00/prg/snpeff/snpEff/snpEff.jar'
+
+refs:
+  ref_fasta: '/home/u00wsg00/work3/ref/hg19/hg19.fasta'
+  # targets_file: '/home/u00wsg00/work3/ref/source/TruSeq_exome_targeted_regions_hg19_chr.bed'
+  # target_flanks_file: '/home/u00wsg00/work3/ref/hg19/targetregion100_hg19_nimblegen_exome_v2.interval_list'
+  dbsnp:
+    ref: '/home/u00wsg00/work3/ref/source/Mills_and_1000G_gold_standard.indels.hg19.sites.vcf'
+    indels: '/home/u00wsg00/work3/ref/source/Mills_and_1000G_gold_standard.indels.hg19.sites.vcf'
+    version: '135'
+  snpeff:
+    config: '/home/u00wsg00/prg/snpeff/snpEff/snpEff.config'
+    db: 'hg19'
+
+####### Sample Infomation
+
+# rgid is "read group id" used by bwa and picard
+# PU:barcode. PU is required by Picard. If not multiplexing sequencing put something like "NA" or "lane1"..
+# PATH for RAW Data Directories
+samples:
+  -
+    name: F193
+    rgid: Project_F193
+    pu: 'NA'
+    path: '/home/u00wsg00/work3/sample/fastq/'
+
+#DATA_TYPE= (TILESQSEQ | TILESFASTQ | TILESELAND | ONEFASTQ | ONEFASTQSE) Single-End
+#TARGET_COVERAGE= (AGILENT | NIMBLEGEN)
+info:
+  data_type: "TILESFASTQ"
+  target_coverage: "NIMBLEGEN"
+  pl: 'ILLumina'
+  lb: "nimblegen"
+  cn: "CPD"
+  ds: "2x100"
+  dt: "2015-11-06"

metadata

@@ -0,0 +1,157 @@
+--- !ruby/object:Gem::Specification
+name: cagnut_core
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Shi-Gang Wang
+- Tse-Ching Ho
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-11-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: tilt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Computational and Analytical Gear for Nucleic acid Utilitarian Techniques
+email:
+- seanwang@goldenio.com
+- tsechingho@goldenio.com
+executables:
+- cagnut
+- stat
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- cagnut_core.gemspec
+- exe/cagnut
+- exe/stat
+- lib/cagnut.rb
+- lib/cagnut/cli/new_project.rb
+- lib/cagnut/cli/pipeline.rb
+- lib/cagnut/configuration.rb
+- lib/cagnut/configuration/base.rb
+- lib/cagnut/configuration/checks/datasets.rb
+- lib/cagnut/configuration/checks/tools.rb
+- lib/cagnut/job_manage.rb
+- lib/cagnut/version.rb
+- lib/cagnut_core.rb
+- templates/Gemfile
+- templates/system.yml
+homepage: https://github.com/CAGNUT/cagnut_core
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Computational and Analytical Gear for Nucleic acid Utilitarian Techniques
+test_files: []