scbi_fqbin 0.2.2 → 0.2.3

data/lib_fqbin_src/fq.c

@@ -0,0 +1,165 @@
+
+#include "lib_fqbin.h"
+#include <stdio.h>
+#include <ctype.h>
+
+#include <unistd.h>
+
+
+// process a fastq file adding it to fbin file
+int iterate_fastq(char *fname, int only_extras, int output_fasta, int output_extras)
+{
+
+    // allocate strings
+    char *name;
+    if ((name = malloc(MAXSEQNAME)) == NULL) {
+        puts("Memory allocation error!");
+        return EXIT_FAILURE;
+    }
+    
+    char *fasta;
+    if ((fasta = malloc(MAXSEQLENGTH)) == NULL) {
+        puts("Memory allocation error!");
+        return EXIT_FAILURE;
+    }
+    
+    char *qual;
+    if ((qual = malloc(MAXSEQLENGTH)) == NULL) {
+        puts("Memory allocation error!");
+        return EXIT_FAILURE;
+    }
+    
+    char *comments;
+    if ((comments = malloc(MAXSEQLENGTH)) == NULL) {
+        puts("Memory allocation error!");
+        return EXIT_FAILURE;
+    }
+    
+    static time_t curr_time=0;
+    static time_t prev_time=0;
+    
+    prev_time=time(NULL);
+    
+    FILE *fastq_file=NULL;
+    
+    
+    int valid=0;
+    int res=0;
+    int r=0;
+    // Open fasta and qual files
+    if (strcmp(fname,"-")==0){
+        fastq_file=stdin;
+    }else{
+
+        open_file(fname,&fastq_file);
+    }
+    
+    if (fastq_file==NULL){
+        printf("TRESb\n");
+    }
+        
+    // for each sequence on fastq file
+    while (valid=get_next_seq_fastq(fastq_file,&name,&fasta,&qual,&comments)){
+        if(valid==1)
+        {
+            r++;
+
+                if (!only_extras){
+                     if (output_fasta){
+                         printf(">%s %s\n%s\n", name, comments, fasta);
+                     }else{
+                         printf("@%s %s\n%s\n", name,comments, fasta);
+                       printf("+%s\n%s\n",name,qual);
+                     }
+                }
+                
+                // if ((extras!=NULL) && (output_extras)) printf ("EXTRAS:%s\n",extras);
+            
+        }else{
+            fprintf(stderr,"Invalid sequence found %s. Aborting import.\n",name);
+            res=-1;
+            break;
+        }
+        
+        // if ((r%10000)==0) {
+        // }
+        
+    }
+
+    curr_time=time(NULL);    
+    printf("\nEnd fastq processing. %d seqs in %.0f s. Rate: %8.2f seqs/s\n",r,difftime(curr_time,prev_time),r/difftime(curr_time,prev_time));
+
+    // free mem
+    free(name);
+    free(fasta);
+    free(qual);
+    free(comments);
+    
+    // close files
+    fclose(fastq_file);
+    
+    return res;
+}
+
+void usage(){
+    printf("Usage: fq [-f][-e][-E] fbin_file seq_name\n\n");
+    printf("    -f    Output sequence in fasta format\n");
+    printf("    -e    Output extras for sequence\n");
+    printf("    -E    Output only extras for sequence\n");
+    
+    exit(-1);
+    
+}
+
+
+
+/*******************************************************/
+/* main                                                */
+/*******************************************************/
+int main(int argc, char *argv[])
+{
+    
+    int ch;
+
+    int output_fasta = 0;
+    int output_extras = 0;
+    int only_extras = 0;
+
+    while ((ch = getopt(argc, argv, "feEh")) != -1) {
+            switch (ch) {
+            case 'f':
+                    output_fasta = 1;
+                    break;
+            case 'e':
+                    output_extras = 1;
+                    break;
+            case 'E':
+                    output_extras = 1;
+                    only_extras = 1;
+                    break;
+
+            case 'h':
+                    usage();
+                    break;
+            case '?':
+            default:
+                    usage();
+            }
+    }
+
+    argc -= optind;
+    argv += optind;
+    
+    
+  // check params
+  if (argc<1)
+  {
+    usage();
+    return -1;
+  }
+
+  int res=iterate_fastq(argv[0],only_extras, output_fasta, output_extras);
+
+  return res;
+}
+

data/lib_fqbin_src/hash_fqbin.c

@@ -0,0 +1,212 @@
+
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include <zlib.h>
+#include <zlib.h>
+#include <stdlib.h>
+
+// Maximum file name (including .idx)
+#define MAXFNAME 512
+
+// Maximum lenght of the name of a sequence
+#define MAXSEQNAME 1024
+#define MAXSEQLENGTH 150000000
+#define DEBUG 1
+#define FALSE 0 
+#define TRUE 1 
+
+#define INVALID_FASTQ_FORMAT -5
+#define INVALID_FASTA_FORMAT -6
+
+#define SEQ_METADATA 10000         
+
+
+long long last_chunk_file(char *filename){
+  
+  // open file again to annotate chunk
+  int file=open(filename,O_APPEND);
+
+  //goto end of file
+  long long pos=lseek(file,0,SEEK_END);
+  if (pos==-1)  {fprintf(stderr,"error %d seeking file %s :%s\n",errno,filename,strerror(errno));return -1;}
+
+  close(file);
+  
+  return pos;
+}
+
+// creates a hash from an index file with the desired chunk size. Chunk size can be 
+// adjusted to fit a good compromise between access speed and used space.
+int hash_index_file(char *filename, int chunk_size, int skip_sort)
+{
+
+  char hash_file_name[MAXFNAME];
+  char indexname[MAXFNAME];
+  char sorted_indexname[MAXFNAME];
+  
+  char sname[MAXSEQNAME];// sequence name
+  long long beginH, gz_chunk=0;
+  char tmp[SEQ_METADATA];
+  int res=0;
+  int error;
+
+  // to save min, max sequences and current chunk
+  char min_name[MAXSEQNAME];
+  char max_name[MAXSEQNAME];
+  long long current_chunk=0;
+  long long count=0;
+  
+
+  strcpy(min_name,"");
+  strcpy(max_name,"");
+
+	// calc index and hash name
+  snprintf(indexname,MAXFNAME,"%s.index",filename);
+  snprintf(hash_file_name,MAXFNAME,"%s.index.hash",filename);
+  
+  // sort index file by external  command
+  if(skip_sort==0)
+  {
+      char cmd[10000];
+      snprintf(cmd,10000,"sort_index %s",indexname);
+      system(cmd);
+  }
+
+  // use sorted index
+  snprintf(sorted_indexname,MAXFNAME,"%s.index.sorted",filename);
+  
+  // open hash file
+  gzFile gzhash_file=gzopen(hash_file_name,"wb");
+
+  // open sorted index file
+  gzFile gzsorted_file_index=gzopen(sorted_indexname,"r");
+  
+  // open output index file
+  // int file_index=open(indexname,flags,0644);
+  gzFile gzfile_index=gzopen(indexname,"w");
+  
+  // write header
+  gzprintf(gzfile_index,"UMACOMPRESSEDFORMAT 1 0 0 999999999999 999999999999\n");
+
+  //reopen
+  gzclose(gzfile_index);
+  gzfile_index=gzopen(indexname,"ab");
+
+
+  if (gzsorted_file_index==NULL) {
+	fprintf(stderr,"error opening gzsorted_file_index :%s\n",gzerror(gzsorted_file_index,&error));
+	return -2;
+  }
+  
+  if (gzfile_index==NULL) {
+	fprintf(stderr,"error opening gzfile_index :%s\n",gzerror(gzfile_index,&error));
+	return -2;
+  }
+  
+  if (gzhash_file==NULL) {
+  	fprintf(stderr,"error opening gzhash_file :%s\n",gzerror(gzhash_file,&error));
+  	return -2;
+  }
+  
+  // podria leerse saltando linea 1, y luego leyendo 10000 lineas sin sscanf
+  
+  // repeat until EOF
+  while ( gzgets(gzsorted_file_index,tmp,sizeof(tmp))!=Z_NULL ) {
+    
+    // parse string
+    sscanf(tmp,"%s %lld %lld",sname,&gz_chunk,&beginH);
+
+    if(strcmp(sname,"UMACOMPRESSEDFORMAT")!=0) // valid index line
+    {
+        
+        // clear chunk_data if any
+        // if (gz_chunk!=current_chunk){
+        if ((count%chunk_size)==0){
+            if (strcmp(min_name,"")!=0){
+                // there is data to write
+                res=gzprintf(gzhash_file,"%s %s %lld\n",min_name,max_name,current_chunk);
+            }
+            
+            strcpy(min_name,"");
+            strcpy(max_name,"");
+            // current_chunk=gz_chunk;
+            current_chunk = gztell(gzfile_index);
+
+            //reopen new gzchunk
+            gzclose(gzfile_index);
+            current_chunk = last_chunk_file(indexname);
+            gzfile_index=gzopen(indexname,"ab");
+
+        }
+         
+        // write line to current gzchunk in index
+        gzprintf(gzfile_index,tmp);
+        
+        // save min_name
+    	if((strcmp(min_name,"")==0) || (strcmp(sname,min_name)<0))
+    	{
+            // replace min_name
+            strcpy(min_name,sname);
+    	}
+	
+    	//save max_name
+    	if((strcmp(max_name,"")==0) || (strcmp(sname,max_name)>0))
+    	{
+            strcpy(max_name,sname);
+      }
+        
+      count++;
+    }
+        
+  }
+  
+  if (strcmp(min_name,"")!=0){
+      // there are data to write
+      res=gzprintf(gzhash_file,"%s %s %lld\n",min_name,max_name,current_chunk);
+  }
+  
+  // close files
+  gzclose(gzhash_file);
+  gzclose(gzfile_index);
+  gzclose(gzsorted_file_index);
+  
+  return 0;
+}
+
+
+/*******************************************************/
+/* main                                                */
+/*******************************************************/
+int main(int argc, char *argv[])
+{
+  // check params
+  if (argc<2)
+  {
+    printf("Usage %s fqbin_file [chunk_size [--skip_sort]]\n\n",argv[0]);
+    return -1;
+  }
+  
+  int chunk_size=10000;
+  int skip_sort=0;
+  
+  if(argc==3){
+      chunk_size=atoi(argv[2]);
+  }
+  
+  if (argc==4){
+      skip_sort=1;
+  }
+  
+  int res=hash_index_file(argv[1],chunk_size, skip_sort);
+  
+  return res;
+}
+

data/lib_fqbin_src/idx_fqbin

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+if [ ! -e "$1" ]; then
+    echo "File $1 does not exists"
+    echo "Usage: $0 fqbin_file.fqbin"
+    exit
+fi
+
+
+if [ -e "$1.index" ]; then
+    echo "Index file $1.index already exists"
+    exit
+fi
+
+echo "Backing up file as $1.old"
+mv $1 $1.old
+
+echo "Regenerating index"
+iterate_fqbin $1.old | mk_fqbin -i -o $1
+rm $1
+mv $1.old $1

data/lib_fqbin_src/iterate_fqbin.c

@@ -0,0 +1,136 @@
+#include "lib_fqbin.h"
+#include <stdio.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+
+void usage(){
+    
+    printf("Usage: iterate_fqbin [-F|-q] fqbin_file\n\n");
+    printf("By default outputs in fastq format\n\n");
+    printf("    -F    Output only sequence in fasta format\n");
+    printf("    -q    Output only qualities in phred format\n\n");
+    printf("    -e    Output only extras\n\n");    
+    
+    exit(-1);
+    
+}
+
+int print_file(struct file_data *filed, int only_fasta, int only_qual, int only_extras){
+    char *sname=NULL;
+    char *fasta=NULL;
+    char *qual=NULL;
+    char *extras=NULL;
+    
+    int len=0;
+    int i=0;
+    // int size=5000;
+    int res=0;
+    
+    
+    while ((res=read_data_sequential(filed, &sname, &fasta, &qual, &extras))==0)
+    {
+
+            if (only_fasta){
+                
+                printf(">%s %s\n", sname, extras);
+                len=strlen(fasta);
+                i=0;
+                for(i = 0; i < len; i+=70)
+                {
+                    printf("%.70s\n", fasta+i);
+                }
+            }else if (only_qual){
+                printf(">%s %s\n", sname, extras);
+                if (qual!=NULL){
+                    len=strlen(qual);
+                    i=0;
+                    for(i = 0; i < len; i++)
+                    {
+                        printf("%02d ", qual[i]-33);
+                        if (((i+1)%30 == 0) || (i==len-1)) printf("\n");
+                    }
+                }
+                
+            }else if (only_extras){
+                printf(">%s %s\n", sname, extras);
+                if (extras!=NULL) printf ("%s\n",extras);
+            }else{
+                printf("@%s %s\n%s\n", sname, extras, fasta);
+                         printf("+\n%s\n",qual);
+                            // printf("+%s\n%s\n",sname,qual);
+            }
+            
+
+    	if ( fasta!=NULL ) {free(fasta);fasta=NULL;}
+    	if ( qual!=NULL ) {free(qual);qual=NULL;}
+    	if ( extras!=NULL ) {free(extras);extras=NULL;}
+    }
+    
+    return res;
+}
+
+
+
+/*******************************************************/
+/* main                                                */
+/*******************************************************/
+int main(int argc, char *argv[])
+{
+
+    //gzFile gzf_bin;
+    // struct file_data filed;
+
+    struct file_data *filed=NULL;
+
+    int ch;
+
+    int output_fasta = 0;
+    int output_qual = 0;
+    int output_extras = 0;
+
+    while ((ch = getopt(argc, argv, "Fqeh")) != -1) {
+            switch (ch) {
+            case 'F':
+                    output_fasta = 1;
+                    break;
+            case 'q':
+                    output_qual=1;
+                    break;
+            case 'e':
+                    output_extras=1;
+                    break;
+            case 'h':
+                    usage();
+                    break;
+            case '?':
+            default:
+                    usage();
+            }
+    }
+
+    argc -= optind;
+    argv += optind;
+    // printf("argc: %d", argc);
+    // printf("argv: %s", argv[0]);
+
+    if (argc!=1)
+    {
+        usage();
+    }
+
+    // initialize reads
+    if (initialize_sequential_reads(&filed, argv[0])==-1){
+        printf("File %s does not exists",argv[0]);
+        exit(-1);
+    }
+
+    int res=print_file(filed,output_fasta,output_qual,output_extras);
+
+    close_sequential_reads(filed);
+
+    return res;
+}
+