bio-gngm 0.2.0 → 0.2.1

data/examples/gngm_qtl_mapping_HTML_maker_set_options.rb

@@ -0,0 +1,243 @@
+#!/usr/bin/env ruby
+
+# Author : Naveed Ishaque (inspired by Dan Maclean) edited again by Dan to include ChD value setting and deletion of SNPs file
+# naveed.ishaque@tsl.ac.uk; naveed.ishaque@hotmail.co.uk
+# Date: 20th June 2012 and 1st November 2012
+
+# This scripts produces a HTML with embedded images showing the SNP density and chastity plots for a given BAM file
+# It will automatically iterate over all contigs, form begining to end
+# NOTE - run using ruby executable: /home/programs/gngm/ruby/bin/ruby
+# NOTE - this will only run on the cluster via a bsub command
+
+
+require 'rubygems'
+require 'bio'
+require 'bio-gngm'
+require 'base64'
+require 'getoptions'
+
+usage = "\n#{$PROGRAM_NAME} reads in a fasta and bam files and produces a html file indicating QTL locations as peaks\n\n\n\t #{$PROGRAM_NAME}\n\n\n -f [reference fasta file]\n -b [bam file]\n -e expected ChD (allele freq) default 1\n -c control ChD (allele freq) default 0.5\n -s List of known SNPS [tab delimited file]\n\n"
+
+# PARSE INPUTS
+
+opt = GetOptions.new(%w(h help f=@s b=@s e=@s c=@s s=@s))
+
+puts "#{usage}" if opt[:h]
+exit if opt[:h]
+puts "#{usage}" if opt[:help]
+exit if opt[:help]
+
+puts "ERROR - no fasta file provided (-f)\n#{usage}" unless opt[:f]
+exit unless opt[:f]
+puts "ERROR - fasta file '#{opt[:f][0]}' does not exist\n#{$usage}" unless FileTest.exist?("#{opt[:f][0]}")
+exit unless FileTest.exist?("#{opt[:f][0]}")
+warn "\nUsing FASTA file #{opt[:f][0]}"
+
+puts "ERROR - no bam file provided (-b)\n#{usage}" unless opt[:b]
+exit unless opt[:b]
+puts "ERROR - BAM file '#{opt[:b][0]}' does not exist\n#{usage}" unless FileTest.exist?("#{opt[:b][0]}")
+exit unless FileTest.exist?("#{opt[:b][0]}")
+warn "Using BAM file #{opt[:b][0]}"
+
+
+expected_chd = 1.0
+expected_chd =opt[:e][0].to_f if expected_chd and opt[:e]
+
+control_chd = 0.5
+control_chd = opt[:c][0].to_f if control_chd and opt[:c]
+
+known_snps = Hash.new { |h,k| h[k] = Array.new }
+if opt[:s].first
+  File.open(opt[:s].first).each  do |line|
+    chr,pos = line.split("\t")
+    known_snps[chr] << pos.to_i
+  end
+end
+
+
+$stderr.puts "using expected ChD: #{expected_chd} and control ChD: #{control_chd}"
+
+hist_bins = [100000, 250000, 500000]
+ks = [5, 7, 9, 11]
+kadjusts = [0.5, 0.25, 0.1, 0.05, 0.01]
+warn "Histogram bin sizes: #{hist_bins}\nThread clusters (K): #{ks}\nKernal adjusts: #{kadjusts}\n"
+
+# LOAD FASTA and find contigs
+sequences = Bio::DB::FastaLengthDB.new(:file => "#{opt[:f][0]}")
+
+
+# For Each contig in the fasta file analyse...
+
+sequences.each do |id,length|
+  warn "\nProcessing #{id}:1 - #{length}..."
+
+  warn "Skipping #{id} as too short ..." if length < (4 * hist_bins.max)
+  next if length < (4 * hist_bins.max)
+
+  g = Bio::Util::Gngm.new(:file => "#{opt[:b][0]}", 
+                 :format => :bam, 
+                 :fasta => "#{opt[:f][0]}",
+                 :start => 1,
+                 :stop => 10000,
+                 :chromosome => id, 
+                 :samtools => {
+                               :q => 20,
+                               :Q => 20
+                              },
+                 :ignore_file => "#{opt[:s][0]}",
+                 :write_pileup => "pileup.txt",
+                 :write_vcf => "snps.vcf"
+    )
+
+  # predict SNPs
+
+  warn "  Prediciting SNPs for #{id}:1-#{length}..."
+  g.snp_positions
+
+          #delete SNPs from known snp_list
+          #a = g.snp_positions.dup
+          #known_snps[seq.entry_id].each {|snp_pos| a.delete_if{|x| x.first == snp_pos} }
+          #$stderr.puts "deleted #{g.snp_positions.length - a.length} snps appearing in #{opt[:s]}"
+          #g.snp_positions = a
+
+
+
+  # produce SNP density histograms
+
+  warn "  Iterating over different histogram bin sizes..."
+  hist_bins.each do |bin_width|
+    warn "  Makings PNG for bin size #{bin_width}..."
+    file_name = "#{id}_SNP_histogram_bin#{bin_width}.png"
+    g.frequency_histogram("#{file_name}",bin_width, :title => "#{id}: SNP density histogram (bin width - #{bin_width})", :width => 1066, :height => 300)
+  end
+
+  # Write to embedded HTML  
+  
+  htmlout = File.open("#{id}.html", 'w') 
+  htmlout.puts "<html>\n"
+  htmlout.puts "  <head>\n"
+  htmlout.puts "    <title>GNGM #{id} - QTL mapping</title>\n"
+  htmlout.puts "    <style type=\"text/css\">\n"
+  htmlout.puts "      table,\n"
+  htmlout.puts "      td,\n"
+  htmlout.puts "      tbody,\n"
+  htmlout.puts "      thead,\n"
+  htmlout.puts "      thead th,\n"
+  htmlout.puts "      tr.even,\n"
+  htmlout.puts "      tr.odd {\n"
+  htmlout.puts "        border: 0;\n"
+  htmlout.puts "      }\n"
+  htmlout.puts "    </style>\n"
+  htmlout.puts "  </head>\n"
+  htmlout.puts "  <body>\n\t\t"
+  htmlout.puts "    <table>\n"
+  htmlout.puts "      <tr>\n"
+  hist_bins.each do |bin_width|
+    htmlout.puts "        <td>\n"
+    htmlout.puts "<img src=\"data:image/gif;base64,"  
+    htmlout.puts [open("#{id}_SNP_histogram_bin#{bin_width}.png").read].pack("m")
+    File.delete("#{id}_SNP_histogram_bin#{bin_width}.png")
+    htmlout.puts "\" width=\"533\" height=\"150\"/>\n"
+    htmlout.puts "        </td>\n"
+  end
+  htmlout.puts "      </tr>\n"
+  htmlout.puts "    </table>\n"
+
+  # Perform chastity calculations 
+
+  warn "  Collecting threads..."
+  g.collect_threads
+  warn "  Iterating over k and kernel adjusts..."
+  ks.each do | k |
+    begin
+      warn "  Makings PNG for k = #{k} ..."
+      warn "    Calculating threads ..."
+      g.calculate_clusters(:k => k, :adjust => 0.5, :control_chd => control_chd, :expected_chd => expected_chd)
+      warn "    Drawing threads ..."
+      filename = "#{id}_k#{k}_threads.png"  
+      g.draw_threads(filename, :title => "#{id}: Chastity bands - all phases (k=#{k})", :width => 700, :height => 300)
+      warn "    Clustering bands ..."
+      filename = "#{id}_k#{k}_clustered_bands.png"
+      g.draw_bands(filename, :title => "#{id}: Homozygous and heterozygous chastity belts (k=#{k})", :width => 800, :height => 300)
+      kadjusts.each do |kernel_adjust|
+        begin
+          warn "    Calculating threads (with kernal adjust #{kernel_adjust}) ..."
+          g.calculate_clusters(:k => k, :adjust => kernel_adjust, :control_chd => control_chd, :expected_chd => expected_chd)
+          warn "    Calculating signal ..."
+          filename = "#{id}_k#{k}_kadjust#{kernel_adjust}_signal.png"
+          g.draw_signal(filename, :title => "#{id}: Homo/Het signal ratio (k=#{k}, kernal=#{kernel_adjust})", :width => 800, :height => 300)
+          warn "    Estimating peaks ..."
+          filename = "#{id}_k#{k}_kadjust#{kernel_adjust}_peaks.png"
+          g.draw_peaks(filename, :title => "#{id}: Signal peaks (k=#{k}, kernal=#{kernel_adjust})", :width => 800, :height => 300)
+        rescue => e
+          $stderr.puts "skipping #{k} #{kernel_adjust} => #{e}"
+        end
+      end
+    rescue => e
+      $stderr.puts "Skipping #{k} => #{e}"
+    end
+  end
+
+  g.close
+
+  # Write to embedded HTML  
+
+  htmlout.puts "    <table>\n"
+
+  # all bands
+  htmlout.puts "      <tr>\n"
+  ks.each do | k | 
+    htmlout.puts "        <td>\n"
+    htmlout.puts "<img src=\"data:image/gif;base64,"  
+    htmlout.puts [open("#{id}_k#{k}_threads.png").read].pack("m")
+    File.delete("#{id}_k#{k}_threads.png")
+    htmlout.puts "\" width=\"400\" height=\"150\"/>"
+    htmlout.puts "        </td>\n"
+  end
+  htmlout.puts "      </tr>\n"
+
+  # homo/het bands
+  htmlout.puts "      <tr>\n"
+  ks.each do | k | 
+    htmlout.puts "        <td>\n"
+    htmlout.puts "<img src=\"data:image/gif;base64,"  
+    htmlout.puts [open("#{id}_k#{k}_clustered_bands.png").read].pack("m")
+    File.delete("#{id}_k#{k}_clustered_bands.png")
+    htmlout.puts "\" width=\"400\" height=\"150\"/>"
+    htmlout.puts "        </td>\n"
+  end
+  htmlout.puts "      </tr>\n"
+
+  # k/adjusts
+  kadjusts.each do |kernel_adjust| 
+    htmlout.puts "      <tr>\n"
+    ks.each do | k |
+      htmlout.puts "        <td>\n"
+      htmlout.puts "<img src=\"data:image/gif;base64,"  
+      htmlout.puts [open("#{id}_k#{k}_kadjust#{kernel_adjust}_signal.png").read].pack("m")
+      File.delete("#{id}_k#{k}_kadjust#{kernel_adjust}_signal.png")
+      htmlout.puts "\" width=\"400\" height=\"150\"/>"
+      htmlout.puts "        </td>\n"
+    end
+    htmlout.puts "      </tr>\n"
+    htmlout.puts "      <tr>\n"
+    ks.each do | k |
+      htmlout.puts "        <td>\n"
+      htmlout.puts "<img src=\"data:image/gif;base64,"  
+      htmlout.puts [open("#{id}_k#{k}_kadjust#{kernel_adjust}_peaks.png").read].pack("m")
+      File.delete("#{id}_k#{k}_kadjust#{kernel_adjust}_peaks.png")
+      htmlout.puts "\" width=\"400\" height=\"150\"/>"
+      htmlout.puts "        </td>\n"
+    end
+    htmlout.puts "      </tr>\n"
+  end
+
+  htmlout.puts "    </table>\n"
+  htmlout.puts "\n  </body>\n</html>\n"
+
+  htmlout.close
+
+end
+
+
+

data/examples/pileup.txt

@@ -0,0 +1,22 @@
+Chr1	23	C	4	AA..	!!II
+Chr1	30	C	5	.-1T.-1T...	!!HII
+Chr1	33	T	5	.-1G.-1G...	!!IHI
+Chr1	37	T	6	AA....	!!IIII
+Chr1	40	T	7	CC.....	!!IHIID
+Chr1	44	T	7	AA.....	!!IIIFB
+Chr1	711	T	16	,.,,,..,,,.,,CC.	>GIIDH;IIIHID48I
+Chr1	1584	G	24	,,.,..a,,,....,,.C,,.,.^].	IIIHIC.IIDGGII=DI'IFHIIE
+Chr1	2544	T	34	,,,,,c,,,...,,..,,.,,,..,,.,.,.,C.	EIGEG&GIIIHIGG@IGGIDIHIIHEIHIIG=/D
+Chr1	3975	G	22	.,....+1T......,...,-1t.,-1t..,	HIHIIGGHDIEGFIFDCIDGBE
+Chr1	4290	C	9	,.,.,,-2at,-2at.-2AT.	IGDIC@<II
+Chr1	4292	T	11	,.,.,***.C^]C	F4DI+!!!I!!
+Chr1	5699	C	25	tT.,,..,.....,..,......,.	!);IH<AHD>;H;I09IDIGIIIGI
+Chr1	5927	T	20	.,......,g,.,..g,...	HI?IHIHII%IBDI=%HBHF
+Chr1	5932	A	20	,......,g,.,..t,....	>-HHIIIH&II7I=2HHHGH
+Chr1	6324	T	23	,$,,+1a,+1a.+1A.+1A.+1A.+1A,+1a,+1a.+1A,+1a,+1a,+1a,+1a.+1A,+1a,+1a,+1a..+1A.+1A.+1A	BII1IIGIIFFIHDFGDIHF?4I
+Chr1	7361	T	28	,,,,,.$,,,,,..C,..,,A.,,,...^],	HIFBIA?IIDHFF(GBIHH,.HBHI.GE
+Chr1	7562	A	31	..,.c.,,,,.,.t,,.,.......,.,...	EFEE+>EEDGC@I)>AI<I?IHIGI8H=IID
+Chr1	8003	C	21	.$,,..,.......-1A....,^]a^],^]a	DIGHIGIAIGGEHIIIF7!0!
+Chr1	8017	G	16	.A....A...,,,,,.	A-<>76**1@BIGIII
+Chr1	9066	G	28	.,,,.,.,,..,..,,.,,,c..,,C.^],	DGHIHIGD2IGII;HHGI?I%IIIH(ID
+Chr1	9971	C	19	,....,,,,-2at.,.-2AT,-2at,,,...	HI=HIGAGIGIIDI<IGIH

data/examples/snps.vcf

@@ -0,0 +1,22 @@
+Chr1	23	.	C	A,C	0	0	DP=4	GT:GQ:DP	0/1:0:4
+Chr1	30	.	C	.	0	0	DP=5	GT:GQ:DP	0/0:0:5
+Chr1	33	.	T	.	0	0	DP=5	GT:GQ:DP	0/0:0:5
+Chr1	37	.	T	.	0	0	DP=6	GT:GQ:DP	0/0:0:6
+Chr1	40	.	T	.	0	0	DP=7	GT:GQ:DP	0/0:0:7
+Chr1	44	.	T	.	0	0	DP=7	GT:GQ:DP	0/0:0:7
+Chr1	711	.	T	.	0	0	DP=16	GT:GQ:DP	0/0:0:16
+Chr1	1584	.	G	.	0	0	DP=24	GT:GQ:DP	0/0:0:24
+Chr1	2544	.	T	.	0	0	DP=34	GT:GQ:DP	0/0:0:34
+Chr1	3975	.	G	.	0	0	DP=22	GT:GQ:DP	0/0:0:22
+Chr1	4290	.	C	.	0	0	DP=9	GT:GQ:DP	0/0:0:9
+Chr1	4292	.	T	.	0	0	DP=11	GT:GQ:DP	0/0:0:11
+Chr1	5699	.	C	.	0	0	DP=25	GT:GQ:DP	0/0:0:25
+Chr1	5927	.	T	.	0	0	DP=20	GT:GQ:DP	0/0:0:20
+Chr1	5932	.	A	.	0	0	DP=20	GT:GQ:DP	0/0:0:20
+Chr1	6324	.	T	.	0	0	DP=23	GT:GQ:DP	0/0:0:23
+Chr1	7361	.	T	.	0	0	DP=28	GT:GQ:DP	0/0:0:28
+Chr1	7562	.	A	.	0	0	DP=31	GT:GQ:DP	0/0:0:31
+Chr1	8003	.	C	.	0	0	DP=21	GT:GQ:DP	0/0:0:21
+Chr1	8017	.	G	.	0	0	DP=16	GT:GQ:DP	0/0:0:16
+Chr1	9066	.	G	.	0	0	DP=28	GT:GQ:DP	0/0:0:28
+Chr1	9971	.	C	.	0	0	DP=19	GT:GQ:DP	0/0:0:19

data/lib/bio-gngm.rb

@@ -10,4 +10,3 @@
 # was ever to get merged into the main bioruby tree.
 
 require 'bio/util/bio-gngm'
-require 'bio/util/mutation_effects'

data/lib/bio/util/bio-gngm.rb

@@ -227,14 +227,36 @@ link:images/signal.png
 
   g = Bio::Util::Gngm.new(:file => "aln.sorted.bam", 
                :format => :bam, 
-               :fasta => "reference.fasta", 
-               :samtools => {:r => "chr1:1-100000",
+               :fasta => "reference.fasta",
+               :start => 100,
+               :stop => 200,
+               :write_pileup => "my_pileup_file.pileup",
+               :write_vcf => "my_vcf_file.vcf",
+               :ignore_file => "my_known_snps.txt" 
+               :samtools => {
                              :q => 20,
                              :Q => 50
                },
                :min_non_ref_freq => 0.5,
-               :min_non_ref => 3
+               :min_non_ref => 3,
+               :start => 1,
+               :stop => 100000,
+               :chromosome => "Chr1",
+               :variant_call => {
+                 :indels => false,  
+                 :min_depth => 6, 
+                 :max_depth => 250, 
+                 :mapping_quality => 20.0, 
+                 :min_non_ref_count => 2, 
+                 :ignore_reference_n => true,
+                 :min_snp_quality => 20,
+                 :min_consensus_quality => 20,
+                 :substitutions => ["C:T","G:A"] 
+                 }
+               
+               
     )
+    
     g.snp_positions
     g.collect_threads(:start => 0.2, :stop => 1.0, :slide => 0.01, :size => 0.1 )
     [0.25, 0.5, 1.0].each do |kernel_adjust| # loop through different kernel values
@@ -311,10 +333,7 @@ The following R packages are required
 Thanks very much indeed to Ryan Austin, who invented NGM in the first place and was very forthcoming with R code, around which this implementation is based.
 
 == Using bio-gngm
-The package is not yet released, a gem will be prepared soon. Until then scripts run fine when saved in the package scripts from within the package directory  with the below pre-amble at the top of the script. Run scripts from the root of the package directory.
-  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-  $LOAD_PATH.unshift(File.dirname(__FILE__))
-  require 'bio-samtools'
+
   require 'bio-gngm'
   
 == API
@@ -335,30 +354,37 @@ class Gngm
   #             :samtools => {:q => 20, :Q => 50}, 
   #             :fasta => "reference.fa"
   #             :start => 100,
-  #             :stop => 200
+  #             :stop => 200,
+  #             :write_pileup => "my_pileup_file.pileup",
+  #             :write_vcf => "my_vcf_file.vcf",
+  #             :ignore_file => "my_known_snps.txt"
+  #
   #  )
   #
   #Required parameters and defaults:
-  #- <tt>:file => nil</tt> -the path to the bam file containing the alignments, a .bai index must be present
-  #- <tt>:format => :bam</tt> -either :bam, :emap, :pileup (pileup expected to be 10 col format from samtools -vcf)
+  #- <tt>:file => nil</tt> -the path to the bam file containing the alignments, a .bai index must be present. A pileup file, or tab-delimited text file can be used.
+  #- <tt>:format => :bam</tt> -either :bam, :pileup, :txt (pileup expected to be 10 col format from samtools -vcf)
   #- <tt>:chromosome => "nil"</tt> -sequence id to look at
   #- <tt>:start => nil</tt> -start position on that sequence
   #- <tt>:stop => nil</tt> -stop position on that sequence
   #- <tt>:fasta => nil</tt> -the path to the FASTA formatted reference sequence
-  #- <tt>:samtools => {:q => 20, :Q => 50}</tt> -options for samtools, see bio-samtools documentation for further details. The :r option is required to specify the region of interest 
+  #- <tt>:write_pileup => false</tt> -the path to a file. SNPs will be written in pileup to this file (indels not output)
+  #- <tt>:write_vcf => false</tt> -the path to a file. SNPs will be written in VCF to this file (indels not output)
+  #- <tt>:ignore_file => false</tt> -file of SNPs in format "reference sequence id \t position \t mapping line nucleotide identity \t reference line nucleotide identity". All SNPs in this file will be ignored
+  #- <tt>:samtools => {:q => 20, :Q => 50}</tt> -options for samtools, see bio-samtools documentation for further details.
   #Optional parameters and defaults:
-  
+  #
   #Most of these are parameters for specific methods and can be over-ridden when particular methods are called
-  #- <tt>:variant_call => {:indels => false,
-  #                         :min_depth => 2, 
-  #                         :max_depth => 10000000, 
-  #                         :min_snp_quality => 20, 
-  #                         :mapping_quality => 10.0, 
-  #                         :min_non_ref_count => 2, 
-  #                         :ignore_reference_n => true, 
-  #                         :min_consensus_quality => 20, 
-  #                         :min_snp_quality => 20 }</tt>. 
-  #                         For Pileup files from old samtools pileup -vcf <tt>:min_consensus_quality</tt> can be applied
+  #- <tt>:variant_call => {:indels => false,</tt>
+  #- <tt>                  :min_depth => 2, </tt>
+  #- <tt>                  :max_depth => 10000000, </tt>
+  #- <tt>                  :min_snp_quality => 20, </tt>
+  #- <tt>                  :mapping_quality => 10.0, </tt>
+  #- <tt>                  :min_non_ref_count => 2, </tt>
+  #- <tt>                  :ignore_reference_n => true, </tt>
+  #- <tt>                  :min_consensus_quality => 20, </tt>
+  #- <tt>                  :min_snp_quality => 20 }</tt>. 
+  # - <tt>                For Pileup files from old samtools pileup -vcf <tt>:min_consensus_quality</tt> can be applied
   #- <tt>:threads => {:start => 0.2, :stop => 1.0, :slide => 0.01, :size => 0.1 }</tt> -options for thread windows
   #- <tt>:insert_size_opts => {:ref_window_size => 200, :ref_window_slide => 50, :isize => 150}</tt> -options for insert size calculations
   #- <tt>:histo_bin_width => 250000</tt> -bin width for histograms of SNP frequency
@@ -385,9 +411,12 @@ class Gngm
       :fasta => nil,
       :samtools => {:q => 20, :Q => 50},
       :indels => false,
-      ##:indels = call indels too, causes return of vcf
+      :write_pileup => false,
+      :write_vcf => false,
+      :ignore_file => false,
       :insert_size_opts => {:ref_window_size => 200, :ref_window_slide => 50, :isize => 150},
-      :variant_call => { :min_depth => 2, 
+      :variant_call => { :indels => false,
+                         :min_depth => 2, 
                          :max_depth => 10000000, 
                          :mapping_quality => 10.0, 
                          :min_non_ref_count => 2, 
@@ -412,6 +441,31 @@ class Gngm
     }
     @opts.merge!(options)
     @opts[:samtools][:r] = "#{options[:chromosome]}:#{options[:start]}-#{options[:stop]}"
+    @pileup_outfile, @vcf_outfile = nil,nil
+    if @opts[:variant_call][:indels] and (@opts[:write_pileup] or @opts[:write_vcf])
+      $stderr.puts "Cannot yet output VCF/Pileup when generating INDELs. Turning output off."
+      @opts[:write_pileup] = false
+      @opts[:write_vcf] = false
+    end
+    if @opts[:write_pileup]
+      @pileup_outfile = File.open(@opts[:write_pileup], "w")
+    end
+    if @opts[:write_vcf]
+      @vcf_outfile = File.open(@opts[:write_vcf], "w")
+    end
+    
+    @known_snps = Hash.new
+    if @opts[:ignore_file]
+      File.open(@opts[:ignore_file], "r").each do |line|
+        col = line.chomp.split(/\t/)
+        if @known_snps[col[0]]
+          @known_snps[col[0]][col[1].to_i] = 1
+        else
+          @known_snps[col[0]] = Hash.new
+          @known_snps[col[0]][col[1].to_i] = 1
+        end
+      end
+    end
     open_file
   end
   
@@ -455,10 +509,8 @@ class Gngm
   #- <tt>:max_depth => 10000000</tt> -maximum quality passing depth of coverage at a position for a SNP call
   #- <tt>:mapping_quality => 10.0</tt> -minimum mapping quality required for a read to be used in depth calculation
   #- <tt>:min_non_ref_count => 2</tt> -minimum number of reads not matching the reference for SNP to be called
-  #- <tt>:ignore_reference_n => true</tt> -ignore positions where the reference is N or n
-  #- <tt>:shore_map => false</tt> -use SHOREmap INTERVAL calculations as described in Anderson et al., 2009, Nature Methods 6. 8. Requires a file of known SNPs between the mapping line (eg Ler in Andersen et al.,) and a reference line (eg Col in Andersen et al).  
-  #- <tt>:snp_file => -file of known SNPs in format "reference sequence id \t position \t mapping line nucleotide identity \t reference line nucleotide identity". Only used when +:shore_map+ is set to true. Only SNPs listed in this file will be considered.
-  #When INDEL calling only one of <tt>:indels, :deletions_only, :insertions_only</tt> should be used. If all are +false+, SNPs are called.
+  #- <tt>:ignore_reference_n => true</tt> -ignore positions where the reference is N or n 
+  #When INDEL calling only one of <tt>:indels</tt> should be used. If +false+, SNPs are called.
   #
   #calculates or returns the value of the instance variable @snp_positions. Only gets positions the first time it is called, in subsequent calls pre-computed positions and statistics are returned, so changing parameters has no effect.
   def snp_positions(optsa={})
@@ -495,7 +547,10 @@ class Gngm
     
     if not @opts[:samtools][:g]
       @file.mpileup(@opts[:samtools]) do |pileup|
-        arr << [pileup.pos, pileup.discordant_chastity] if pileup.is_snp?(opts) and is_allowed_substitution?(pileup.ref_base, pileup.consensus,opts)
+       if pileup.is_snp?(opts) and is_allowed_substitution?(pileup.ref_base, pileup.consensus,opts) and not @known_snps[pileup.ref_name][pileup.pos]
+         arr << [pileup.pos, pileup.discordant_chastity]
+         write(pileup)
+       end
       end
     else
       @file.mpileup_plus(@opts[:samtools]) do |vcf|
@@ -503,9 +558,9 @@ class Gngm
         next if (opts[:ignore_reference_n] and vcf.ref =~ /N/i)
         ##indel use returns the vcf allele_frequency, not the ChDs (because calculating it is a mess... )
         if opts[:indels]
-          arr << [vcf.pos, vcf.non_ref_allele_freq] if vcf.is_indel?(opts) and is_allowed_substitution?(vcf.ref, vcf.alt,opts)
+          arr << [vcf.pos, vcf.non_ref_allele_freq] if vcf.is_indel?(opts) and is_allowed_substitution?(vcf.ref, vcf.alt,opts) and not @known_snps[vcf.ref][vcf.pos]
         else
-          arr << [vcf.pos, vcf.non_ref_allele_freq] if vcf.is_snp?(opts) and is_allowed_substitution?(vcf.ref, vcf.alt,opts)
+          arr << [vcf.pos, vcf.non_ref_allele_freq] if vcf.is_snp?(opts) and is_allowed_substitution?(vcf.ref, vcf.alt,opts) and not @known_snps[vcf.ref][vcf.pos]
         end
       end
     end
@@ -515,11 +570,6 @@ class Gngm
     arr
   end
   
-  private
-  def get_snp_positions_from_map(options={})
-    arr = []
-    opts = @opts[:variant_call].merge(options)
-  end
   
   #this does not filter snps, other than to check they are in the right region and are allowed substitutions.. no qual control, assumed to be done prior
   #text file is of format chr\tpos\tref\talt\tfreq\n
@@ -530,7 +580,7 @@ class Gngm
         chr,pos,ref,alt,freq = line.chomp.split("\t")
         pos = pos.to_i
         freq = freq.to_f
-        next unless chr == @opts[:chromosome] and pos >= @opts[:start] and pos <= @opts[:stop] and is_allowed_substitution?(ref,alt,opts)
+        next unless chr == @opts[:chromosome] and pos >= @opts[:start] and pos <= @opts[:stop] and is_allowed_substitution?(ref,alt,opts) and not @known_snps[chr][pos]
         arr << [pos, freq]
     end
     @snp_positions = arr
@@ -546,13 +596,25 @@ class Gngm
        next
      end 
       #old fashioned 10 col pileup format has extra fields we can use if needed
-     if pileup.is_snp?(opts) and not pileup.consensus_quality.nil? and not pileup.snp_quality.nil?
+     if pileup.is_snp?(opts) and not pileup.consensus_quality.nil? and not pileup.snp_quality.nil? and not @known_snps[pileup.ref_name][pileup.pos]
+       write(pileup)
        arr << [pileup.pos, pileup.discordant_chastity] if pileup.consensus_quality > opts[:min_consensus_quality] and pileup.snp_quality > opts[:min_snp_quality] and is_allowed_substitution?(pileup.ref_base, pileup.consensus,opts)
      end
     end
     @snp_positions = arr
   end
   
+  private
+  #writes out pileup/vcf files of SNPs that were used
+  def write(obj)
+    if @opts[:write_pileup] 
+      @pileup_outfile.puts(obj.to_s)
+    end
+    if @opts[:write_vcf]
+      @vcf_outfile.puts(obj.to_vcf)
+    end
+  end
+  
   private
   #Gets the insert size for each alignment in the BAM positions from a BAM file according to quality criteria passed by Bio::Util::Gngm#get_insert_size_frequency.
   def get_insert_size_frequency_from_bam(opts={})