bio-vcf 0.8.1 → 0.8.2

data/ragel/gen_vcfheaderline_parser.rl

@@ -0,0 +1,122 @@
+# Ragel lexer for VCF-header
+#
+# This is a partial lexer for the VCF header format. Bio-vcf uses this
+# to generate meta information in (for example) JSON format. The
+# advantage of using a full state engine is that it allows for easy
+# parsing of key-value pairs with syntax checking and, for example,
+# escaped quotes in quoted string values. This edition validates ID and
+# Number fields only.
+#
+# Note the .rb version is generated from ./ragel/gen_vcfheaderline_parser.rl
+
+module BioVcf
+
+  module VcfHeaderParser
+
+    module RagelKeyValues
+  
+=begin
+%%{
+
+  machine simple_lexer;
+  
+  action mark { ts=p }
+  action endquoted {
+    emit.call(:value,data,ts,p)
+  }
+
+  action kw {
+    emit.call(:kw,data,ts,p)
+  }
+  
+  squote = "'";
+  dquote = '"';
+  not_squote_or_escape = [^'\\];
+  not_dquote_or_escape = [^"\\];
+  escaped_something = /\\./;
+  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
+  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;
+
+  integer     = ('+'|'-')?digit+;
+  float       = ('+'|'-')?digit+'.'digit+;
+  assignment  = '=';
+  identifier  = (alpha alnum+); 
+  str         = (ss|dd)* ;       
+  boolean     = '.';
+  key_word    = ( ('Type'|'Description'|'Source'|'Version'|identifier - ('ID'|'Number')) >mark %{ emit.call(:key_word,data,ts,p) } );
+  any_value   = ( str|( integer|float|boolean|identifier >mark %{ emit.call(:value,data,ts,p) } ));
+  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );
+  
+  number_value = ( ( integer|boolean|'A'|'R'|'G' ) >mark %{ emit.call(:value,data,ts,p) } );
+
+  id_kv     = ( ( ('ID') %kw '=' id_value ) @!{ error_code="ID"} );
+  number_kv = ( ( ('Number') %kw '=' number_value ) @!{ error_code="Number"} );
+  key_value = ( id_kv | number_kv | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };
+  
+  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
+}%%
+=end
+
+%% write data;
+# %% this just fixes our syntax highlighting...
+
+def self.run_lexer(buf, options = {})
+  do_debug = (options[:debug] == true)
+  data = buf.unpack("c*") if(buf.is_a?(String))
+  eof = data.length
+  values = []
+  stack = []
+
+  emit = lambda { |type, data, ts, p|
+    # Print the type and text of the last read token
+    # p ts,p
+    puts "#{type}: #{data[ts...p].pack('c*')}" if do_debug
+    values << [type,data[ts...p].pack('c*')]
+  }
+
+  error_code = nil
+  
+  %% write init;
+  %% write exec;
+
+  raise "ERROR: "+error_code+" in "+buf if error_code
+
+  begin
+    res = {}
+    # p values
+    values.each_slice(2) do | a,b |
+      # p '*',a,b
+      res[a[1]] = b[1]
+      # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
+    end
+  rescue
+    print "ERROR: "
+    p values
+    raise
+  end
+  p res if do_debug
+  res
+end
+    end
+  end 
+end
+
+if __FILE__ == $0
+
+lines = <<LINES
+##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
+##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth",Extra="Yes?">
+##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
+##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
+##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf",Source="dbsnp",Version="138">
+##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
+##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
+##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
+LINES
+
+lines.strip.split("\n").each { |s|
+  print s,"\n"
+  p BioVcf::VcfHeaderParser::RagelKeyValues.run_lexer(s, debug: false)
+}
+
+end # test

data/ragel/generate.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+
+ragel -R gen_vcfheaderline_parser.rl ; ruby gen_vcfheaderline_parser.rb
+
+cp gen_vcfheaderline_parser.rb ../lib/bio-vcf/vcfheader_line.rb

data/template/vcf2json_full_header.erb

@@ -0,0 +1,23 @@
+=HEADER
+<% require 'json' %>
+[
+  {
+    "HEADER": {
+      "options":  <%= options.to_h.to_json %>,
+      "files":    <%= ARGV %>,
+      "version":  "<%= BIOVCF_VERSION %>",
+      },
+    "COLUMNS": <%= header.column_names.to_json %>,
+    "META": <%= header.meta.to_json %>
+  },
+=BODY
+
+{
+  "seq:chr": "<%= rec.chrom %>" ,
+  "seq:pos": <%= rec.pos %> ,
+  "seq:ref": "<%= rec.ref %>" ,
+  "seq:alt": "<%= rec.alt[0] %>"
+  <% if rec.info.dp %> , "dp": <%= rec.info.dp %> <% end %>
+},
+=FOOTER
+]

data/template/vcf2json_use_meta.erb

@@ -0,0 +1,41 @@
+=HEADER
+<% require 'json' %>
+[
+  {
+    "HEADER": {
+      "options":  <%= options.to_h.to_json %>,
+      "files":    <%= ARGV %>,
+      "version":  "<%= BIOVCF_VERSION %>",
+      },
+    "COLUMNS": <%= header.column_names.to_json %>,
+    "META": <%= header.meta.to_json %>
+  },
+=BODY
+<% sample_num = 0
+   sample_name = nil
+   sample_size = header.samples.size
+%>
+{
+  "seq:chr": "<%= rec.chrom %>" ,
+  "seq:pos": <%= rec.pos %> ,
+  "seq:ref": "<%= rec.ref %>" ,
+  "seq:alt": "<%= rec.alt[0] %>"
+  <% if rec.info.dp %> , "dp": <%= rec.info.dp %> <% end %>,
+  {
+  <% rec.each_sample do |s| %>
+    <% if not s.empty? 
+    
+         sample_name = header.samples[sample_num]
+         sample_num += 1
+    %>
+    "<%= sample_name %>": {
+      <% header.meta['FORMAT'].each_key do |k| %>
+        "<%= k %>": <%= s[k].to_json %>,
+      <% end %>
+    } <%= (sample_num<sample_size ? "," : "") %>
+    <% end %>
+  <% end %>
+  }
+},
+=FOOTER
+]

data/test/data/regression/vcf2json_full_header.ref

@@ -0,0 +1,261 @@
+
+[
+  {
+    "HEADER": {
+      "files":    [],
+      },
+    "COLUMNS": ["CHROM","POS","ID","REF","ALT","QUAL","FILTER","INFO","FORMAT","Original","s1t1","s2t1","s3t1","s1t2","s2t2","s3t2"],
+    "META": {"INFO":{"AC":{"ID":"AC","Number":"A","Type":"Integer","Description":"Allele count in genotypes, for each ALT allele, in the same order as listed"},"AF":{"ID":"AF","Number":"A","Type":"Float","Description":"Allele Frequency, for each ALT allele, in the same order as listed"},"AN":{"ID":"AN","Number":"1","Type":"Integer","Description":"Total number of alleles in called genotypes"},"BaseQRankSum":{"ID":"BaseQRankSum","Number":"1","Type":"Float","Description":"Z-score from Wilcoxon rank sum test of Alt Vs. Ref base qualities"},"DP":{"ID":"DP","Number":"1","Type":"Integer","Description":"Approximate read depth; some reads may have been filtered"},"DS":{"ID":"DS","Number":"0","Type":"Flag","Description":"Were any of the samples downsampled?"},"Dels":{"ID":"Dels","Number":"1","Type":"Float","Description":"Fraction of Reads Containing Spanning Deletions"},"FS":{"ID":"FS","Number":"1","Type":"Float","Description":"Phred-scaled p-value using Fisher's exact test to detect strand bias"},"HaplotypeScore":{"ID":"HaplotypeScore","Number":"1","Type":"Float","Description":"Consistency of the site with at most two segregating haplotypes"},"InbreedingCoeff":{"ID":"InbreedingCoeff","Number":"1","Type":"Float","Description":"Inbreeding coefficient as estimated from the genotype likelihoods per-sample when compared against the Hardy-Weinberg expectation"},"MLEAC":{"ID":"MLEAC","Number":"A","Type":"Integer","Description":"Maximum likelihood expectation (MLE) for the allele counts (not necessarily the same as the AC), for each ALT allele, in the same order as listed"},"MLEAF":{"ID":"MLEAF","Number":"A","Type":"Float","Description":"Maximum likelihood expectation (MLE) for the allele frequency (not necessarily the same as the AF), for each ALT allele, in the same order as listed"},"MQ":{"ID":"MQ","Number":"1","Type":"Float","Description":"RMS Mapping Quality"},"MQ0":{"ID":"MQ0","Number":"1","Type":"Integer","Description":"Total Mapping Quality Zero Reads"},"MQRankSum":{"ID":"MQRankSum","Number":"1","Type":"Float","Description":"Z-score From Wilcoxon rank sum test of Alt vs. Ref read mapping qualities"},"QD":{"ID":"QD","Number":"1","Type":"Float","Description":"Variant Confidence/Quality by Depth"},"RPA":{"ID":"RPA","Number":".","Type":"Integer","Description":"Number of times tandem repeat unit is repeated, for each allele (including reference)"},"RU":{"ID":"RU","Number":"1","Type":"String","Description":"Tandem repeat unit (bases)"},"ReadPosRankSum":{"ID":"ReadPosRankSum","Number":"1","Type":"Float","Description":"Z-score from Wilcoxon rank sum test of Alt vs. Ref read position bias"},"STR":{"ID":"STR","Number":"0","Type":"Flag","Description":"Variant is a short tandem repeat"}},"FORMAT":{"AD":{"ID":"AD","Number":".","Type":"Integer","Description":"Allelic depths for the ref and alt alleles in the order listed"},"DP":{"ID":"DP","Number":"1","Type":"Integer","Description":"Approximate read depth (reads with MQ=255 or with bad mates are filtered)"},"GQ":{"ID":"GQ","Number":"1","Type":"Integer","Description":"Genotype Quality"},"GT":{"ID":"GT","Number":"1","Type":"String","Description":"Genotype"},"PL":{"ID":"PL","Number":"G","Type":"Integer","Description":"Normalized, Phred-scaled likelihoods for genotypes as defined in the VCF specification"}},"fileformat":"VCFv4.1","FILTER":"<ID=LowQual,Description=\"Low quality\">","GATKCommandLine":"<ID=UnifiedGenotyper,Version=2.8-1-g932cd3a,Date=\"Sat Jan 25 10:33:56 CET 2014\",Epoch=1390642436187,CommandLineOptions=\"analysis_type=UnifiedGenotyper input_file=[/data_fedor12/BAM/sander/Liver_clones/BIOPSY17513D/mapping/BIOPSY17513D_dedup_realigned_recalibrated.bam, /data_fedor12/BAM/sander/Liver_clones/clone3/mapping/clone3_dedup_realigned_recalibrated.bam, /data_fedor12/BAM/sander/Liver_clones/clone4/mapping/clone4_dedup_realigned_recalibrated.bam, /data_fedor12/BAM/sander/Liver_clones/clone10/mapping/clone10_dedup_realigned_recalibrated.bam, /data_fedor12/BAM/sander/Liver_clones/subclone33/mapping/subclone33_dedup_realigned_recalibrated.bam, /data_fedor12/BAM/sander/Liver_clones/subclone46/mapping/subclone46_dedup_realigned_recalibrated.bam, /data_fedor12/BAM/sander/Liver_clones/subclone105/mapping/subclone105_dedup_realigned_recalibrated.bam] read_buffer_size=null phone_home=AWS gatk_key=null tag=NA read_filter=[] intervals=[/data_fedor13/sander/variant_calling/Liver_clones/.queue/scatterGather/UnifiedGenotyper_noref-1-sg/temp_001_of_500/scatter.intervals] excludeIntervals=null interval_set_rule=UNION interval_merging=ALL interval_padding=0 reference_sequence=/data_fedor13/common_data/references/H_sapiens/GATK_b37_bundle_reference/basespace/human_g1k_v37.fasta nonDeterministicRandomSeed=false disableDithering=false maxRuntime=-1 maxRuntimeUnits=MINUTES downsampling_type=BY_SAMPLE downsample_to_fraction=null downsample_to_coverage=250 baq=OFF baqGapOpenPenalty=40.0 fix_misencoded_quality_scores=false allow_potentially_misencoded_quality_scores=false useOriginalQualities=false defaultBaseQualities=-1 performanceLog=null BQSR=null quantize_quals=0 disable_indel_quals=false emit_original_quals=false preserve_qscores_less_than=6 globalQScorePrior=-1.0 allow_bqsr_on_reduced_bams_despite_repeated_warnings=false validation_strictness=SILENT remove_program_records=false keep_program_records=false sample_rename_mapping_file=null unsafe=null disable_auto_index_creation_and_locking_when_reading_rods=false num_threads=1 num_cpu_threads_per_data_thread=1 num_io_threads=0 monitorThreadEfficiency=false num_bam_file_handles=null read_group_black_list=null pedigree=[] pedigreeString=[] pedigreeValidationType=STRICT allow_intervals_with_unindexed_bam=false generateShadowBCF=false variant_index_type=DYNAMIC_SEEK variant_index_parameter=-1 logging_level=INFO log_to_file=null help=false version=false genotype_likelihoods_model=SNP pcr_error_rate=1.0E-4 computeSLOD=false annotateNDA=false pair_hmm_implementation=LOGLESS_CACHING min_base_quality_score=17 max_deletion_fraction=0.05 allSitePLs=false min_indel_count_for_genotyping=5 min_indel_fraction_per_sample=0.25 indelGapContinuationPenalty=10 indelGapOpenPenalty=45 indelHaplotypeSize=80 indelDebug=false ignoreSNPAlleles=false allReadsSP=false ignoreLaneInfo=false reference_sample_calls=(RodBinding name= source=UNBOUND) reference_sample_name=null sample_ploidy=2 min_quality_score=1 max_quality_score=40 site_quality_prior=20 min_power_threshold_for_calling=0.95 min_reference_depth=100 exclude_filtered_reference_sites=false output_mode=EMIT_VARIANTS_ONLY heterozygosity=0.001 indel_heterozygosity=1.25E-4 genotyping_mode=DISCOVERY standard_min_confidence_threshold_for_calling=30.0 standard_min_confidence_threshold_for_emitting=30.0 alleles=(RodBinding name= source=UNBOUND) max_alternate_alleles=6 input_prior=[] contamination_fraction_to_filter=0.0 contamination_fraction_per_sample_file=null p_nonref_model=EXACT_INDEPENDENT exactcallslog=null dbsnp=(RodBinding name= source=UNBOUND) comp=[] out=org.broadinstitute.sting.gatk.io.stubs.VariantContextWriterStub no_cmdline_in_header=org.broadinstitute.sting.gatk.io.stubs.VariantContextWriterStub sites_only=org.broadinstitute.sting.gatk.io.stubs.VariantContextWriterStub bcf=org.broadinstitute.sting.gatk.io.stubs.VariantContextWriterStub onlyEmitSamples=[] debug_file=null metrics_file=null annotation=[] excludeAnnotation=[] filter_reads_with_N_cigar=false filter_mismatching_base_and_quals=false filter_bases_not_stored=false\">","contig":"<ID=GL000192.1,length=547496,assembly=b37>","reference":"file:human_g1k_v37.fasta"}
+  },
+{
+  "seq:chr": "1" ,
+  "seq:pos": 10257 ,
+  "seq:ref": "A" ,
+  "seq:alt": "C"
+   , "dp": 1518 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 10291 ,
+  "seq:ref": "C" ,
+  "seq:alt": "T"
+   , "dp": 1433 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 10297 ,
+  "seq:ref": "C" ,
+  "seq:alt": "T"
+   , "dp": 1440 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 10303 ,
+  "seq:ref": "C" ,
+  "seq:alt": "T"
+   , "dp": 1460 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 10315 ,
+  "seq:ref": "C" ,
+  "seq:alt": "T"
+   , "dp": 1500 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 10321 ,
+  "seq:ref": "C" ,
+  "seq:alt": "T"
+   , "dp": 1537 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 10327 ,
+  "seq:ref": "T" ,
+  "seq:alt": "C"
+   , "dp": 1641 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 10583 ,
+  "seq:ref": "G" ,
+  "seq:alt": "A"
+   , "dp": 129 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 10665 ,
+  "seq:ref": "C" ,
+  "seq:alt": "G"
+   , "dp": 28 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 10694 ,
+  "seq:ref": "C" ,
+  "seq:alt": "G"
+   , "dp": 22 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 10723 ,
+  "seq:ref": "C" ,
+  "seq:alt": "G"
+   , "dp": 18 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 12783 ,
+  "seq:ref": "G" ,
+  "seq:alt": "A"
+   , "dp": 939 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 13116 ,
+  "seq:ref": "T" ,
+  "seq:alt": "G"
+   , "dp": 721 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 13118 ,
+  "seq:ref": "A" ,
+  "seq:alt": "G"
+   , "dp": 703 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 13178 ,
+  "seq:ref": "G" ,
+  "seq:alt": "A"
+   , "dp": 929 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 13302 ,
+  "seq:ref": "C" ,
+  "seq:alt": "T"
+   , "dp": 692 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 13757 ,
+  "seq:ref": "G" ,
+  "seq:alt": "A"
+   , "dp": 1149 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 13868 ,
+  "seq:ref": "A" ,
+  "seq:alt": "G"
+   , "dp": 1108 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 13896 ,
+  "seq:ref": "C" ,
+  "seq:alt": "A"
+   , "dp": 830 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 14354 ,
+  "seq:ref": "C" ,
+  "seq:alt": "A"
+   , "dp": 764 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 14464 ,
+  "seq:ref": "A" ,
+  "seq:alt": "T"
+   , "dp": 809 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 14673 ,
+  "seq:ref": "G" ,
+  "seq:alt": "C"
+   , "dp": 754 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 14699 ,
+  "seq:ref": "C" ,
+  "seq:alt": "G"
+   , "dp": 719 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 14907 ,
+  "seq:ref": "A" ,
+  "seq:alt": "G"
+   , "dp": 1083 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 14930 ,
+  "seq:ref": "A" ,
+  "seq:alt": "G"
+   , "dp": 1092 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 14933 ,
+  "seq:ref": "G" ,
+  "seq:alt": "A"
+   , "dp": 1089 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 14948 ,
+  "seq:ref": "G" ,
+  "seq:alt": "A"
+   , "dp": 1050 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 14976 ,
+  "seq:ref": "G" ,
+  "seq:alt": "A"
+   , "dp": 939 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 15118 ,
+  "seq:ref": "A" ,
+  "seq:alt": "G"
+   , "dp": 1004 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 15190 ,
+  "seq:ref": "G" ,
+  "seq:alt": "A"
+   , "dp": 1090 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 15211 ,
+  "seq:ref": "T" ,
+  "seq:alt": "G"
+   , "dp": 992 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 15274 ,
+  "seq:ref": "A" ,
+  "seq:alt": "G"
+   , "dp": 636 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 15447 ,
+  "seq:ref": "A" ,
+  "seq:alt": "G"
+   , "dp": 1213 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 15688 ,
+  "seq:ref": "C" ,
+  "seq:alt": "T"
+   , "dp": 904 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 16068 ,
+  "seq:ref": "T" ,
+  "seq:alt": "C"
+   , "dp": 398 
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 16103 ,
+  "seq:ref": "T" ,
+  "seq:alt": "G"
+   , "dp": 526 
+},]