genfrag 0.0.0.1

data/lib/genfrag/app/index_command.rb

@@ -0,0 +1,227 @@
+
+module Genfrag
+class App
+
+class IndexCommand < Command
+
+  attr_reader :sizes
+
+# Run from command-line
+#
+  def cli_run( args )
+    parse args
+
+    @input_filenames = ARGV
+
+    validate_options(options)
+
+    if options[:tracktime]
+      Genfrag.tracktime {
+        run(options, @input_filenames, true)
+      }
+    else
+      run(options, @input_filenames, true)
+    end
+
+  end
+
+# Main class for creating the index - accepts multiple input files. Either an SQLite database or
+# a flat file index is created (extension .tdf) which is unique for the input file combination. 
+# This file is used by the Search routine later.
+#
+  def run(ops=@ops, input_filenames=[], cli=false)
+    if ops.kind_of? OpenStruct
+      @ops = ops.dup
+    elsif ops.kind_of? Hash
+      @ops = OpenStruct.new(ops)
+    else
+      raise ArgumentError
+    end
+    
+  # Set defaults
+    @ops.verbose    ||= false
+    @ops.quiet      ||= false
+    @ops.sqlite     ||= false
+    @ops.filelookup ||= nil
+    @ops.filefasta  ||= nil
+    @ops.re5        ||= nil
+    @ops.re3        ||= nil
+    @ops.indir      ||= '.'
+    @ops.outdir     ||= '.'
+
+    @input_filenames = input_filenames.empty? ? [@ops.filefasta] : input_filenames
+    @sizes = {}    
+    db = IndexCommand::DB.new(@ops, @input_filenames)
+    @re5_ds, @re3_ds = [@ops.re5, @ops.re3].map {|x| Bio::RestrictionEnzyme::DoubleStranded.new(x)}
+    db.write_headers    
+
+    if @ops.verbose
+      cli_p(cli, <<-END
+RE5: #{@ops.re5}
+#{@re5_ds.aligned_strands_with_cuts.primary}
+#{@re5_ds.aligned_strands_with_cuts.complement}
+
+RE3: #{@ops.re3}
+#{@re3_ds.aligned_strands_with_cuts.primary}
+#{@re3_ds.aligned_strands_with_cuts.complement}
+END
+)
+    end
+
+  # unit test with aasi, aari, and ppii
+    re5_regexp, re3_regexp = [@ops.re5, @ops.re3].map {|x| Bio::Sequence::NA.new( Bio::RestrictionEnzyme::DoubleStranded.new(x).aligned_strands.primary ).to_re }
+
+    entries = {}
+  # Account for exact duplicate sequences
+    @input_filenames.each do |input_filename|
+      Bio::FlatFile.auto(File.join(@ops.indir, input_filename)).each_entry do |e|
+        e.definition.tr!("\t",'')
+        s = e.seq.to_s.downcase
+        if entries[s]
+          entries[s] << e.definition
+        else
+          entries[s] = [e.definition]
+        end
+      end
+    end
+    
+    a_re = /(.*)(#{re5_regexp})/
+    b_re = /(.*?)(#{re3_regexp})/
+    
+    normalized_fasta_id=0
+    entries.each do |seq, definitions|
+      normalized_fasta_id+=1
+      db.write_entry_to_fasta(normalized_fasta_id, seq, definitions)
+      
+    # NOTE the index command is slow because of the match functions, compare with ruby 1.9
+      m1 = a_re.match(seq)
+      if m1
+      # Find the fragment 'frag1' cut most right in seq with re5_regexp
+        frag1 = $2 + m1.post_match          
+  
+        position = $1.size
+  
+        m2 = b_re.match( frag1 )
+  
+      # Now cut frag1 with re3_regexp resulting in frag2
+        if m2
+          @frag2 = $1 + $2
+          if @ops.verbose
+            cli_p(cli, <<-END
+---
+#{definitions.join("\n")}
+#{@frag2}
+END
+)
+          end
+          @sizes[@frag2.size] ||= []
+          @sizes[@frag2.size] << [position, normalized_fasta_id]
+        end
+      end
+
+    end
+
+    i=0
+    @sizes.each do |size,info|
+      i+=1
+      db.write_entry_to_freq(i, size, info.map {|x| x.join(' ')}.join(', ') )
+    end
+    
+    if @ops.verbose
+      @sizes.each { |@entry| cli_p(cli, @entry.inspect) }
+    else
+      cli_p(cli, "Cut sites found: #{@sizes.values.flatten.size / 2}")
+    end
+    
+    db.close
+  end
+
+
+############
+# Command-line
+############
+
+
+# Option parser for command-line
+#
+  def opt_parser
+    std_opts = standard_options
+
+    opts = OptionParser.new
+    opts.banner = 'Usage: genfrag index [options]'
+
+    opts.separator ''
+    opts.separator "  Create a database of sequence fragments that match the last 5' fragment"
+    opts.separator "  cut by two restricting enzymes RE3 and RE5."
+    opts.separator "  The Fasta file defined by the --fasta option is taken as input."
+    opts.separator "  Two files are created for the search function - a lookup file, and"
+    opts.separator "  the contents of the Fasta file rewritten in a special format. You can"
+    opts.separator "  specify the name of the lookup file with the --lookup option."
+
+    opts.separator ''
+
+    ary = [:verbose, :quiet, :tracktime, :indir, :outdir, :sqlite, :re5, :re3,
+      :filelookup, :filefasta
+    ]
+    ary.each { |a| opts.on(*std_opts[a]) }
+
+    opts.separator ''
+    opts.separator '  Common Options:'
+    opts.on( '-h', '--help', 'show this message' ) { @out.puts opts; exit 1 }
+    opts.separator '  Examples:'
+    opts.separator '    genfrag index -f example.fasta --re5 BstYI --re3 MseI'
+    opts.separator '    genfrag index --out /tmp --in . -f example.fasta --re5 BstYI --re3 MseI'
+    opts
+  end
+
+# Parse options passed from command-line
+#
+  def parse( args )
+    opts = opt_parser
+
+    if args.empty?
+      @out.puts opts
+      exit 1
+    end
+
+  # parse the command line arguments
+    opts.parse! args
+  end
+
+# Validate options passed from the command-line
+  def validate_options(o)
+    if o[:filefasta] == nil
+      clierr_p "missing option: must supply fasta filename"
+      exit 1
+    end
+
+    if o[:re5] == nil
+      clierr_p "missing option: re5"
+      exit 1
+    end
+
+    if o[:re3] == nil
+      clierr_p "missing option: re3"
+      exit 1
+    end
+
+    begin
+      Bio::RestrictionEnzyme::DoubleStranded.new(o[:re3])
+    rescue
+      clierr_p "re3 is not an enzyme name"
+      exit 1
+    end
+
+    begin
+      Bio::RestrictionEnzyme::DoubleStranded.new(o[:re5])
+    rescue
+      clierr_p "re5 is not an enzyme name"
+      exit 1
+    end
+  end
+
+end  # class IndexCommand
+end  # class App
+end  # module Genfrag
+
+# EOF

data/lib/genfrag/app/index_command/db.rb

@@ -0,0 +1,105 @@
+module Genfrag
+class App
+
+class IndexCommand < Command
+
+  class DB    
+    attr_accessor :ops      # an OpenStruct of the options
+    attr_accessor :input_filenames
+    attr_accessor :normalized_fasta
+    attr_accessor :freq_lookup
+
+    def initialize( ops, input_filenames )
+      @normalized_fasta = nil
+      @freq_lookup = nil
+      @ops = ops
+      @input_filenames = input_filenames
+      
+    end
+    
+    def sc
+      @ops.sqlite ? 'sqlite' : 'csv'
+    end
+    
+    
+    def write_headers
+      self.send("write_headers_#{sc}")
+    end
+    
+    def write_headers_sqlite
+      @normalized_fasta = SQLite3::Database.new( File.join(@ops.outdir, Genfrag.name_normalized_fasta(@input_filenames,@ops.filefasta) + '.db') )
+      sql = <<-SQL
+        drop table if exists db_normalized_fasta;
+        create table db_normalized_fasta (
+          id integer,
+          definitions text,
+          sequence text
+        );
+        create unique index db_normalized_fasta_idx on db_normalized_fasta(id);
+      SQL
+      @normalized_fasta.execute_batch( sql )
+      @freq_lookup = SQLite3::Database.new( File.join(@ops.outdir, Genfrag.name_freq_lookup(@input_filenames,@ops.filefasta,@ops.filelookup,@ops.re5,@ops.re3) + '.db') )
+      sql = <<-SQL
+        drop table if exists db_freq_lookup;
+        create table db_freq_lookup (
+        id integer,
+        size integer,
+        positions text
+        );
+        create unique index db_freq_lookup_idx on db_freq_lookup(id);
+      SQL
+      @freq_lookup.execute_batch( sql )
+    end
+    
+    def write_headers_csv
+      @normalized_fasta = File.new(File.join(@ops.outdir,Genfrag.name_normalized_fasta(@input_filenames,@ops.filefasta) + '.tdf'), 'w')
+      @normalized_fasta.puts %w(id Definitions Sequence).join("\t")
+      @freq_lookup = File.new( File.join(@ops.outdir,Genfrag.name_freq_lookup(@input_filenames,@ops.filefasta,@ops.filelookup,@ops.re5,@ops.re3) + '.tdf'), 'w')
+      @freq_lookup.puts %w(id Size Positions).join("\t")
+    end
+    
+    
+    def write_entry_to_fasta(normalized_fasta_id, seq, definitions)
+      self.send("write_entry_to_fasta_#{sc}", normalized_fasta_id, seq, definitions)
+    end
+
+    def write_entry_to_fasta_sqlite(normalized_fasta_id, seq, definitions)
+      @normalized_fasta.execute( "insert into db_normalized_fasta values ( ?, ?, ? )", normalized_fasta_id, CSV.generate_line(definitions), seq )
+    end
+    
+    def write_entry_to_fasta_csv(normalized_fasta_id, seq, definitions)
+      @normalized_fasta.puts [normalized_fasta_id,CSV.generate_line(definitions),seq].join("\t")
+    end
+    
+    
+    def write_entry_to_freq(i, size, str)
+      self.send("write_entry_to_freq_#{sc}", i, size, str)
+    end
+      
+    def write_entry_to_freq_sqlite(i, size, str)
+      @freq_lookup.execute( "insert into db_freq_lookup values ( ?, ?, ? )", i, size, str )
+    end
+    
+    def write_entry_to_freq_csv(i, size, str)
+      @freq_lookup.puts [i,size,str].join("\t")
+    end
+    
+    
+    def close
+      self.send("close_#{sc}")
+    end
+    
+    def close_sqlite
+    end
+    
+    def close_csv
+      @normalized_fasta.close
+      @freq_lookup.close
+    end
+  end
+
+end  # class IndexCommand
+end  # class App
+end  # module Genfrag
+
+# EOF

data/lib/genfrag/app/search_command.rb

@@ -0,0 +1,298 @@
+
+module Genfrag
+class App
+
+class SearchCommand < Command
+
+  def cli_run( args )
+    parse args
+
+    @input_filenames = ARGV
+    input_filenames = [@input_filenames].flatten
+    processed_adapters=nil
+    
+    validate_options(options)
+    
+    
+    if options[:sqlite]
+      processed_fasta_file  = SearchCommand::ProcessFile.process_db_fasta_file( SQLite3::Database.new( Genfrag.name_normalized_fasta(input_filenames,options[:filefasta]) + '.db' ) )
+      processed_freq_lookup = SearchCommand::ProcessFile.process_db_freq_lookup( SQLite3::Database.new( Genfrag.name_freq_lookup(input_filenames,options[:filefasta],options[:filelookup],options[:re5],options[:re3]) + '.db' ) )
+    else
+      processed_fasta_file  = SearchCommand::ProcessFile.process_tdf_fasta_file(  IO.readlines( Genfrag.name_normalized_fasta(input_filenames,options[:filefasta]) + '.tdf' ) )
+      processed_freq_lookup = SearchCommand::ProcessFile.process_tdf_freq_lookup( IO.readlines( Genfrag.name_freq_lookup(input_filenames,options[:filefasta],options[:filelookup],options[:re5],options[:re3]) + '.tdf' ) )
+    end
+    
+      if options[:fileadapters]
+      processed_adapters = SearchCommand::ProcessFile.process_tdf_adapters( IO.readlines( Genfrag.name_adapters(options[:fileadapters]) + '.tdf' ), options[:named_adapter5], options[:named_adapter3] )
+    end
+    
+    run(options, processed_fasta_file, processed_freq_lookup, processed_adapters, true)
+  end
+
+  def opt_parser
+    std_opts = standard_options
+
+    opts = OptionParser.new
+    opts.banner = 'Usage: genfrag search [options]'
+
+    opts.separator ''
+    opts.separator "  Search a database of sequence fragments that match the last 5'"
+    opts.separator "  fragment cut by two restricting enzymes RE3 and RE5, as created by the"
+    opts.separator "  index function. Next, adapters are applied to search a subset of"
+    opts.separator "  fragments, as is used in some protocols."
+
+    opts.separator ''
+    ary = [:verbose, :quiet, :tracktime, :indir, :outdir, :sqlite, :re5, :re3,
+      :filelookup, :filefasta, :fileadapters, :adapter5_sequence, :adapter3_sequence,
+      :adapter5_size, :adapter3_size, :named_adapter5, :named_adapter3,
+      :adapter5, :adapter3
+    ]
+    ary.each { |a| opts.on(*std_opts[a]) }
+        
+    opts.separator ''
+    opts.separator '  Common Options:'
+    opts.on( '-h', '--help', 'show this message' ) { @out.puts opts; exit }
+    
+    opts.separator '  Examples:'
+    opts.separator '    genfrag search -f example.fasta --re5 BstYI --re3 MseI --adapter5 tt'
+    opts.separator '    genfrag search -f example.fasta --re5 BstYI --re3 MseI --add 26 --adapter5 ct --adapter3 aa --size 190,215'
+    opts.separator '    genfrag search -f example.fasta --re5 BstYI --re3 MseI --adapter5-size 11 --adapter5 tt --adapter3-size 15 --size 168'
+    opts.separator '    genfrag search -f example.fasta --re5 BstYI --re3 MseI --adapter5-sequence GACTGCGTAGTGATC --adapter5 tt --adapter3-size 15 --size 168'
+    opts.separator '    genfrag search -f example.fasta --re5 BstYI --re3 MseI --adapter5-size 11 --adapter5 ct --adapter3-size 15 --adapter3 aa --size 190,215'
+    opts.separator '    genfrag search -f example.fasta --re5 BstYI --re3 MseI --add 26 --named-adapter5 BstYI-T4 --named-adapter3 MseI-21 --size 190,215'
+    opts
+  end
+  
+  def parse( args )
+    opts = opt_parser
+
+    if args.empty?
+      @out.puts opts
+      exit 1
+    end
+    
+  # parse the command line arguments
+    opts.parse! args
+    
+  end
+  
+  def validate_options(o)
+    if o[:filefasta] == nil
+      clierr_p "missing option: must supply fasta filename"
+      exit 1
+    end
+    
+    if o[:re5] == nil
+      clierr_p "missing option: re5"
+      exit 1
+    end
+    
+    if o[:re3] == nil
+      clierr_p "missing option: re3"
+      exit 1
+    end
+    
+    begin
+      Bio::RestrictionEnzyme::DoubleStranded.new(o[:re3])
+    rescue
+      clierr_p "re3 is not an enzyme name"
+      exit 1
+    end
+    
+    begin
+      Bio::RestrictionEnzyme::DoubleStranded.new(o[:re5])
+    rescue
+      clierr_p "re5 is not an enzyme name"
+      exit 1
+    end
+  end
+  
+  def run(ops=OpenStruct.new, processed_fasta_file=nil, processed_freq_lookup=nil, processed_adapters=nil, cli=false)
+    if ops.kind_of? OpenStruct
+      @ops = ops.dup
+    elsif ops.kind_of? Hash
+      @ops = OpenStruct.new(ops)
+    else
+      raise ArgumentError
+    end
+    
+  # Set defaults
+    @ops.verbose        ||= false
+    @ops.quiet          ||= false
+    @ops.sqlite         ||= false
+    @ops.re5            ||= nil
+    @ops.re3            ||= nil
+    @ops.size           ||= [0]
+    @ops.adapter5_size  ||= nil
+    @ops.adapter3_size  ||= nil
+    @ops.adapter5       ||= nil
+    @ops.adapter3       ||= nil
+    
+    @sizes = processed_freq_lookup
+    @sequences = processed_fasta_file
+    @adapters = {}
+    @re5_ds, @re3_ds = [@ops.re5, @ops.re3].map {|x| Bio::RestrictionEnzyme::DoubleStranded.new(x)}
+    if @ops.verbose
+      cli_p(cli, <<-END
+RE5: #{@ops.re5}
+#{@re5_ds.aligned_strands_with_cuts.primary}
+#{@re5_ds.aligned_strands_with_cuts.complement}
+
+RE3: #{@ops.re3}
+#{@re3_ds.aligned_strands_with_cuts.primary}
+#{@re3_ds.aligned_strands_with_cuts.complement}
+
+adapter5: #{@ops.adapter5}
+adapter3: #{@ops.adapter3}
+END
+)
+    end
+
+    if @ops.named_adapter5 and @ops.adapter5
+      raise ArgumentError, "Cannot have both 'adapter5' and 'named_adapter5'"
+    elsif @ops.named_adapter3 and @ops.adapter3
+      raise ArgumentError, "Cannot have both 'adapter3' and 'named_adapter3'"
+    end
+    
+    if !processed_adapters and (@ops.named_adapter5 or @ops.named_adapter3)
+      raise ArgumentError, "Must specify --fileadapters when using a named_adapter"
+    end
+    
+    if processed_adapters
+      adapter_setup_1(processed_adapters)
+    else
+      adapter_setup_2
+    end
+    
+  # translated adapter 3' if given in reverse orientation - e.g. _tt is 
+  # translated to aa (reversed) and _tct returns the primary strand
+  # ending in specific 'tct'
+    if @adapters[:adapter3_specificity] =~ /^_/
+      seq3 = Bio::Sequence::NA.new(@adapters[:adapter3_specificity][1..-1]).downcase
+      @adapters[:adapter3_specificity] = seq3.complement.to_s
+    end
+    
+    if @ops.adapter5_size and @ops.adapter5_sequence and (@ops.adapter5_size != @adapters[:adapter5_size])
+      raise ArgumentError, "--adapter5-sequence and --adapter5-size both supplied"
+    end
+    if @ops.adapter3_size and @ops.adapter3_sequence and (@ops.adapter3_size != @adapters[:adapter3_size])
+      raise ArgumentError, "--adapter3-sequence and --adapter3-size both supplied"
+    end
+
+    @trim = calculate_trim_for_nucleotides(@re5_ds, @re3_ds)
+    
+  # ------
+  # Start calculations
+  #
+    left_trim, right_trim = calculate_left_and_right_trims(@trim)
+  
+    matching_fragments = find_matching_fragments(@sizes, left_trim, right_trim)
+    results = []
+   
+    matching_fragments.each do |hit|
+      hit.each do |entry|
+        seq = @sequences[entry[:fasta_id]][:sequence]
+        raw_frag = seq[entry[:offset]..(entry[:offset]+entry[:raw_size]-1)]
+
+        primary_frag, complement_frag = trim_sequences(raw_frag, Bio::Sequence::NA.new(raw_frag).forward_complement, left_trim, right_trim, @trim)
+        
+        p = primary_frag.dup
+        c = complement_frag.dup
+        
+      # note the next two if-statements at this lever chain together with 'p' and 'c'
+        if @adapters[:adapter5_specificity]
+          p, c = matches_adapter(5, p, c, raw_frag, @trim)
+          next if !p  # next if returned false -- no match
+        end
+        
+        if @adapters[:adapter3_specificity]
+          p, c = matches_adapter(3, p, c, raw_frag, @trim)
+          next if !p  # next if returned false -- no match
+        end
+        
+        primary_frag_with_adapters = p
+        complement_frag_with_adapters = c
+        
+        results << {:raw_frag => raw_frag, :primary_frag => primary_frag, :primary_frag_with_adapters => primary_frag_with_adapters, :complement_frag => complement_frag, :complement_frag_with_adapters => complement_frag_with_adapters, :entry => entry, :seq => seq} # FIXME
+      end
+    end
+  
+    if results.size == 0
+      cli_p(cli,"Nothing found") if @ops.verbose
+    end
+
+    sorted_results = {}
+    results.sort {|a,b| a[:seq] <=> b[:seq]}.each do |r|
+      raise "shouldn't happen" if sorted_results[r[:seq]] != nil
+      sorted_results[r[:seq]] = {}
+      x = sorted_results[r[:seq]]
+      x['sequence size'] = r[:seq].size
+      x['fragment - primary strand'] = r[:primary_frag]
+      x['fragment - complement strand'] = r[:complement_frag]
+      x['fragment with adapters - primary strand'] = r[:primary_frag_with_adapters]
+      x['fragment with adapters - complement strand'] = r[:complement_frag_with_adapters]
+    end
+
+    if @ops.verbose
+      ary = ['sequence size', 'fragment - primary strand', 'fragment - complement strand',
+        'fragment with adapters - primary strand', 'fragment with adapters - complement strand']    
+    else
+      ary = ['fragment with adapters - primary strand', 'fragment with adapters - complement strand']    
+    end
+    sorted_results.each do |k,v|
+      cli_p(cli, '---')
+      if @ops.verbose
+        cli_p(cli, '- sequence')
+        cli_p(cli, "  #{k}")
+      end
+
+      ary.each do |a|
+        cli_p(cli, "- #{a}")
+        cli_p(cli, "  #{v[a]}")
+      end
+    end
+
+    return results
+  end
+  
+  def adapter_setup_1(hsh)
+    l = lambda do |i|
+      if @ops.send("adapter#{i}")
+        @adapters["adapter#{i}_specificity".to_sym] = @ops.send("adapter#{i}")
+        if @ops.send("adapter#{i}_sequence")
+          @adapters["adapter#{i}_sequence".to_sym] = @ops.send("adapter#{i}_sequence").gsub(/\|N*$/i,'')
+          @adapters["adapter#{i}_size".to_sym] = @adapters["adapter#{i}_sequence".to_sym].size + @adapters["adapter#{i}_specificity".to_sym].size
+        else
+          @adapters["adapter#{i}_size".to_sym] = @ops.send("adapter#{i}_size")
+        end
+      elsif hsh["adapter#{i}_specificity".to_sym]
+        @adapters["adapter#{i}_specificity".to_sym] = hsh["adapter#{i}_specificity".to_sym]
+        @adapters["adapter#{i}_sequence".to_sym]    = hsh["adapter#{i}_sequence".to_sym]
+        @adapters["adapter#{i}_size".to_sym]        = hsh["adapter#{i}_sequence".to_sym].size + hsh["adapter#{i}_specificity".to_sym].size
+      end
+    end
+  # set adapter 5' and 3' respectively using above procs
+    l.call(5)
+    l.call(3)
+  end
+  
+  def adapter_setup_2
+    l = lambda do |i|
+      @adapters["adapter#{i}_specificity".to_sym] = @ops.send("adapter#{i}")
+      if @ops.send("adapter#{i}_sequence")
+        @adapters["adapter#{i}_sequence".to_sym] = @ops.send("adapter#{i}_sequence").gsub(/\|N*$/i,'')
+        @adapters["adapter#{i}_size".to_sym] = @adapters["adapter#{i}_sequence".to_sym].size + @adapters["adapter#{i}_specificity".to_sym].size
+      else
+        @adapters["adapter#{i}_size".to_sym] = @ops.send("adapter#{i}_size")
+      end
+    end
+    l.call(5)
+    l.call(3)
+  end
+  
+
+end  # class SearchCommand
+end  # class App
+end  # module Genfrag
+
+# EOF