bio-bigwig 0.0.1

data/ext/bio/bigwig/libBigWig/io.c

@@ -0,0 +1,296 @@
+#ifndef NOCURL
+#include <curl/curl.h>
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include "bigWigIO.h"
+#include <inttypes.h>
+#include <errno.h>
+
+size_t GLOBAL_DEFAULTBUFFERSIZE;
+
+#ifndef NOCURL
+uint64_t getContentLength(URL_t *URL) {
+    double size;
+    if(curl_easy_getinfo(URL->x.curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &size) != CURLE_OK) {
+        return 0;
+    }
+    if(size== -1.0) return 0;
+    return (uint64_t) size;
+}
+
+//Fill the buffer, note that URL may be left in an unusable state on error!
+CURLcode urlFetchData(URL_t *URL, unsigned long bufSize) {
+    CURLcode rv;
+    char range[1024];
+
+    if(URL->filePos != (size_t) -1) URL->filePos += URL->bufLen;
+    else URL->filePos = 0;
+
+    URL->bufPos = URL->bufLen = 0; //Otherwise, we can't copy anything into the buffer!
+    sprintf(range,"%lu-%lu", URL->filePos, URL->filePos+bufSize-1);
+    rv = curl_easy_setopt(URL->x.curl, CURLOPT_RANGE, range);
+    if(rv != CURLE_OK) {
+        fprintf(stderr, "[urlFetchData] Couldn't set the range (%s)\n", range);
+        return rv;
+    }
+
+    rv = curl_easy_perform(URL->x.curl);
+    errno = 0; //Sometimes curl_easy_perform leaves a random errno remnant
+    return rv;
+}
+
+//Read data into a buffer, ideally from a buffer already in memory
+//The loop is likely no longer needed.
+size_t url_fread(void *obuf, size_t obufSize, URL_t *URL) {
+    size_t remaining = obufSize, fetchSize;
+    void *p = obuf;
+    CURLcode rv;
+
+    while(remaining) {
+        if(!URL->bufLen) {
+            rv = urlFetchData(URL, URL->bufSize);
+            if(rv != CURLE_OK) {
+                fprintf(stderr, "[url_fread] urlFetchData (A) returned %s\n", curl_easy_strerror(rv));
+                return 0;
+            }  
+        } else if(URL->bufLen < URL->bufPos + remaining) { //Copy the remaining buffer and reload the buffer as needed
+            p = memcpy(p, URL->memBuf+URL->bufPos, URL->bufLen - URL->bufPos);
+            if(!p) return 0;
+            p += URL->bufLen - URL->bufPos;
+            remaining -= URL->bufLen - URL->bufPos;
+            if(remaining) {
+                if(!URL->isCompressed) {
+                    fetchSize = URL->bufSize;
+                } else {
+                    fetchSize = (remaining<URL->bufSize)?remaining:URL->bufSize;
+                }
+                rv = urlFetchData(URL, fetchSize);
+                if(rv != CURLE_OK) {
+                    fprintf(stderr, "[url_fread] urlFetchData (B) returned %s\n", curl_easy_strerror(rv));
+                    return 0;
+                }
+            }
+        } else {
+            p = memcpy(p, URL->memBuf+URL->bufPos, remaining);
+            if(!p) return 0;
+            URL->bufPos += remaining;
+            remaining = 0;
+        }
+    }
+    return obufSize;
+}
+#endif
+
+//Returns the number of bytes requested or a smaller number on error
+//Note that in the case of remote files, the actual amount read may be less than the return value!
+size_t urlRead(URL_t *URL, void *buf, size_t bufSize) {
+#ifndef NOCURL
+    if(URL->type==0) {
+        return fread(buf, bufSize, 1, URL->x.fp)*bufSize;
+    } else {
+        return url_fread(buf, bufSize, URL);
+    }
+#else
+    return fread(buf, bufSize, 1, URL->x.fp)*bufSize;
+#endif
+}
+
+size_t bwFillBuffer(void *inBuf, size_t l, size_t nmemb, void *pURL) {
+    URL_t *URL = (URL_t*) pURL;
+    void *p = URL->memBuf;
+    size_t copied = l*nmemb;
+    if(!p) return 0;
+
+    p += URL->bufLen;
+    if(l*nmemb > URL->bufSize - URL->bufPos) { //We received more than we can store!
+        copied = URL->bufSize - URL->bufLen;
+    }
+    memcpy(p, inBuf, copied);
+    URL->bufLen += copied;
+
+    if(!URL->memBuf) return 0; //signal error
+    return copied;
+}
+
+//Seek to an arbitrary location, returning a CURLcode
+//Note that a local file returns CURLE_OK on success or CURLE_FAILED_INIT on any error;
+CURLcode urlSeek(URL_t *URL, size_t pos) {
+#ifndef NOCURL
+    char range[1024];
+    CURLcode rv;
+
+    if(URL->type == BWG_FILE) {
+#endif
+        if(fseek(URL->x.fp, pos, SEEK_SET) == 0) {
+            errno = 0;
+            return CURLE_OK;
+        } else {
+            return CURLE_FAILED_INIT; //This is arbitrary
+        }
+#ifndef NOCURL
+    } else {
+        //If the location is covered by the buffer then don't seek!
+        if(pos < URL->filePos || pos >= URL->filePos+URL->bufLen) {
+            URL->filePos = pos;
+            URL->bufLen = 0; //Otherwise, filePos will get incremented on the next read!
+            URL->bufPos = 0;
+            //Maybe this works for FTP?
+            sprintf(range,"%lu-%lu", pos, pos+URL->bufSize-1);
+            rv = curl_easy_setopt(URL->x.curl, CURLOPT_RANGE, range);
+            if(rv != CURLE_OK) {
+                fprintf(stderr, "[urlSeek] Couldn't set the range (%s)\n", range);
+                return rv;
+            }
+            rv = curl_easy_perform(URL->x.curl);
+            if(rv != CURLE_OK) {
+                fprintf(stderr, "[urlSeek] curl_easy_perform received an error!\n");
+            }
+            errno = 0;  //Don't propogate remnant resolved libCurl errors
+            return rv;
+        } else {
+            URL->bufPos = pos-URL->filePos;
+            return CURLE_OK;
+        }
+    }
+#endif
+}
+
+URL_t *urlOpen(char *fname, CURLcode (*callBack)(CURL*), const char *mode) {
+    URL_t *URL = calloc(1, sizeof(URL_t));
+    if(!URL) return NULL;
+    char *url = NULL, *req = NULL;
+#ifndef NOCURL
+    CURLcode code;
+    char range[1024];
+#endif
+
+    URL->fname = fname;
+
+    if((!mode) || (strchr(mode, 'w') == 0)) {
+        //Set the protocol
+#ifndef NOCURL
+        if(strncmp(fname, "http://", 7) == 0) URL->type = BWG_HTTP;
+        else if(strncmp(fname, "https://", 8) == 0) URL->type = BWG_HTTPS;
+        else if(strncmp(fname, "ftp://", 6) == 0) URL->type = BWG_FTP;
+        else URL->type = BWG_FILE;
+#else
+        URL->type = BWG_FILE;
+#endif
+
+        //local file?
+        if(URL->type == BWG_FILE) {
+            URL->filePos = -1; //This signals that nothing has been read
+            URL->x.fp = fopen(fname, "rb");
+            if(!(URL->x.fp)) {
+                free(URL);
+                fprintf(stderr, "[urlOpen] Couldn't open %s for reading\n", fname);
+                return NULL;
+            }
+#ifndef NOCURL
+        } else {
+            //Remote file, set up the memory buffer and get CURL ready
+            URL->memBuf = malloc(GLOBAL_DEFAULTBUFFERSIZE);
+            if(!(URL->memBuf)) {
+                free(URL);
+                fprintf(stderr, "[urlOpen] Couldn't allocate enough space for the file buffer!\n");
+                return NULL;
+            }
+            URL->bufSize = GLOBAL_DEFAULTBUFFERSIZE;
+            URL->x.curl = curl_easy_init();
+            if(!(URL->x.curl)) {
+                fprintf(stderr, "[urlOpen] curl_easy_init() failed!\n");
+                goto error;
+            }
+            //Negotiate a reasonable HTTP authentication method
+            if(curl_easy_setopt(URL->x.curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY) != CURLE_OK) {
+                fprintf(stderr, "[urlOpen] Failed instructing curl to use any HTTP authentication it finds to be suitable!\n");
+                goto error;
+            }
+            //Follow redirects
+            if(curl_easy_setopt(URL->x.curl, CURLOPT_FOLLOWLOCATION, 1L) != CURLE_OK) {
+                fprintf(stderr, "[urlOpen] Failed instructing curl to follow redirects!\n");
+                goto error;
+            }
+            //Set the URL
+            if(curl_easy_setopt(URL->x.curl, CURLOPT_URL, fname) != CURLE_OK) {
+                fprintf(stderr, "[urlOpen] Couldn't set CURLOPT_URL!\n");
+                goto error;
+            }
+            //Set the range, which doesn't do anything for HTTP
+            sprintf(range, "0-%lu", URL->bufSize-1);
+            if(curl_easy_setopt(URL->x.curl, CURLOPT_RANGE, range) != CURLE_OK) {
+                fprintf(stderr, "[urlOpen] Couldn't set CURLOPT_RANGE (%s)!\n", range);
+                goto error;
+            }
+            //Set the callback info, which means we no longer need to directly deal with sockets and header!
+            if(curl_easy_setopt(URL->x.curl, CURLOPT_WRITEFUNCTION, bwFillBuffer) != CURLE_OK) {
+                fprintf(stderr, "[urlOpen] Couldn't set CURLOPT_WRITEFUNCTION!\n");
+                goto error;
+            }
+            if(curl_easy_setopt(URL->x.curl, CURLOPT_WRITEDATA, (void*)URL) != CURLE_OK) {
+                fprintf(stderr, "[urlOpen] Couldn't set CURLOPT_WRITEDATA!\n");
+                goto error;
+            }
+            //Ignore certificate errors with https, libcurl just isn't reliable enough with conda
+            if(curl_easy_setopt(URL->x.curl, CURLOPT_SSL_VERIFYPEER, 0) != CURLE_OK) {
+                fprintf(stderr, "[urlOpen] Couldn't set CURLOPT_SSL_VERIFYPEER to 0!\n");
+                goto error;
+            }
+            if(curl_easy_setopt(URL->x.curl, CURLOPT_SSL_VERIFYHOST, 0) != CURLE_OK) {
+                fprintf(stderr, "[urlOpen] Couldn't set CURLOPT_SSL_VERIFYHOST to 0!\n");
+                goto error;
+            }
+            if(callBack) {
+                code = callBack(URL->x.curl);
+                if(code != CURLE_OK) {
+                    fprintf(stderr, "[urlOpen] The user-supplied call back function returned an error: %s\n", curl_easy_strerror(code));
+                    goto error;
+                }
+            }
+            code = curl_easy_perform(URL->x.curl);
+            errno = 0; //Sometimes curl_easy_perform leaves a random errno remnant
+            if(code != CURLE_OK) {
+                fprintf(stderr, "[urlOpen] curl_easy_perform received an error: %s\n", curl_easy_strerror(code));
+                goto error;
+            }
+#endif
+        }
+    } else {
+        URL->type = BWG_FILE;
+        URL->x.fp = fopen(fname, mode);
+        if(!(URL->x.fp)) {
+            free(URL);
+            fprintf(stderr, "[urlOpen] Couldn't open %s for writing\n", fname);
+            return NULL;
+        }
+    }
+    if(url) free(url);
+    if(req) free(req);
+    return URL;
+
+#ifndef NOCURL
+error:
+    if(url) free(url);
+    if(req) free(req);
+    free(URL->memBuf);
+    curl_easy_cleanup(URL->x.curl);
+    free(URL);
+    return NULL;
+#endif
+}
+
+//Performs the necessary free() operations and handles cleaning up curl
+void urlClose(URL_t *URL) {
+    if(URL->type == BWG_FILE) {
+        fclose(URL->x.fp);
+#ifndef NOCURL
+    } else {
+        free(URL->memBuf);
+        curl_easy_cleanup(URL->x.curl);
+#endif
+    }
+    free(URL);
+}

data/ext/bio/bigwig/libBigWig/test/exampleWrite.c

@@ -0,0 +1,76 @@
+#include "bigWig.h"
+
+int main(int argc, char *argv[]) {
+    bigWigFile_t *fp = NULL;
+    char *chroms[] = {"1", "2"};
+    char *chromsUse[] = {"1", "1", "1"};
+    uint32_t chrLens[] = {1000000, 1500000};
+    uint32_t starts[] = {0, 100, 125,
+                         200, 220, 230,
+                         500, 600, 625,
+                         700, 800, 850};
+    uint32_t ends[] = {5, 120, 126,
+                       205, 226, 231};
+    float values[] = {0.0f, 1.0f, 200.0f,
+                      -2.0f, 150.0f, 25.0f,
+                      0.0f, 1.0f, 200.0f,
+                      -2.0f, 150.0f, 25.0f,
+                      -5.0f, -20.0f, 25.0f,
+                      -5.0f, -20.0f, 25.0f};
+    
+
+    if(bwInit(1<<17) != 0) {
+        fprintf(stderr, "Received an error in bwInit\n");
+        return 1;
+    }
+
+    fp = bwOpen("test/example_output.bw", NULL, "w");
+    if(!fp) {
+        fprintf(stderr, "An error occurred while opening example_output.bw for writingn\n");
+        return 1;
+    }
+
+    //Allow up to 10 zoom levels, though fewer will be used in practice
+    if(bwCreateHdr(fp, 10)) goto error;
+
+    //Create the chromosome lists
+    fp->cl = bwCreateChromList(chroms, chrLens, 2);
+    if(!fp->cl) goto error;
+
+    //Write the header
+    if(bwWriteHdr(fp)) goto error;
+
+    //Some example bedGraph-like entries
+    if(bwAddIntervals(fp, chromsUse, starts, ends, values, 3)) goto error;
+    //We can continue appending similarly formatted entries
+    //N.B. you can't append a different chromosome (those always go into different
+    if(bwAppendIntervals(fp, starts+3, ends+3, values+3, 3)) goto error;
+
+    //Add a new block of entries with a span. Since bwAdd/AppendIntervals was just used we MUST create a new block
+    if(bwAddIntervalSpans(fp, "1", starts+6, 20, values+6, 3)) goto error;
+    //We can continue appending similarly formatted entries
+    if(bwAppendIntervalSpans(fp, starts+9, values+9, 3)) goto error;
+
+    //Add a new block of fixed-step entries
+    if(bwAddIntervalSpanSteps(fp, "1", 900, 20, 30, values+12, 3)) goto error;
+    //The start is then 760, since that's where the previous step ended
+    if(bwAppendIntervalSpanSteps(fp, values+15, 3)) goto error;
+
+    //Add a new chromosome
+    chromsUse[0] = "2";
+    chromsUse[1] = "2";
+    chromsUse[2] = "2";
+    if(bwAddIntervals(fp, chromsUse, starts, ends, values, 3)) goto error;
+
+    //Closing the file causes the zoom levels to be created
+    bwClose(fp);
+    bwCleanup();
+
+    return 0;
+
+error:
+    fprintf(stderr, "Received an error somewhere!\n");
+    bwClose(fp);
+    bwCleanup();
+    return 1;
+}

data/ext/bio/bigwig/libBigWig/test/testBigBed.c

@@ -0,0 +1,132 @@
+#include "bigWig.h"
+#include <stdio.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <assert.h>
+
+void bwPrintHdr(bigWigFile_t *bb) {
+    uint64_t i;
+    int64_t i64;
+    printf("Version:    %"PRIu16"\n", bb->hdr->version);
+    printf("Levels:     %"PRIu16"\n", bb->hdr->nLevels);
+    printf("ctOffset:   0x%"PRIx64"\n", bb->hdr->ctOffset);
+    printf("dataOffset: 0x%"PRIx64"\n", bb->hdr->dataOffset);
+    printf("indexOffset:        0x%"PRIx64"\n", bb->hdr->indexOffset);
+    printf("fieldCount: %"PRIu32"\n", bb->hdr->fieldCount);
+    printf("definedFieldCount: %"PRIu32"\n", bb->hdr->definedFieldCount);
+    printf("sqlOffset:  0x%"PRIx64"\n", bb->hdr->sqlOffset);
+    printf("summaryOffset:      0x%"PRIx64"\n", bb->hdr->summaryOffset);
+    printf("bufSize:    %"PRIu32"\n", bb->hdr->bufSize);
+
+    if(bb->hdr->nLevels) {
+        printf("	i	level	data	index\n");
+    }
+    for(i=0; i<bb->hdr->nLevels; i++) {
+        printf("\t%"PRIu64"\t%"PRIu32"\t%"PRIx64"\t%"PRIx64"\n", i, bb->hdr->zoomHdrs->level[i], bb->hdr->zoomHdrs->dataOffset[i], bb->hdr->zoomHdrs->indexOffset[i]);
+    }
+
+    printf("nBasesCovered:      %"PRIu64"\n", bb->hdr->nBasesCovered);
+    printf("minVal:     %f\n", bb->hdr->minVal);
+    printf("maxVal:     %f\n", bb->hdr->maxVal);
+    printf("sumData:    %f\n", bb->hdr->sumData);
+    printf("sumSquared: %f\n", bb->hdr->sumSquared);
+
+    //Chromosome idx/name/length
+    if(bb->cl) {
+        printf("Chromosome List\n");
+        printf("  idx\tChrom\tLength (bases)\n");
+        for(i64=0; i64<bb->cl->nKeys; i64++) {
+            printf("  %"PRIu64"\t%s\t%"PRIu32"\n", i64, bb->cl->chrom[i64], bb->cl->len[i64]);
+        }
+    }
+}
+
+void bwPrintIndexNode(bwRTreeNode_t *node, int level) {
+    uint16_t i;
+    if(!node) return;
+    for(i=0; i<node->nChildren; i++) {
+        if(node->isLeaf) {
+            printf("  %i\t%"PRIu32"\t%"PRIu32"\t%"PRIu32"\t%"PRIu32"\t0x%"PRIx64"\t%"PRIu64"\n", level,\
+                node->chrIdxStart[i], \
+                node->baseStart[i], \
+                node->chrIdxEnd[i], \
+                node->baseEnd[i], \
+                node->dataOffset[i], \
+                node->x.size[i]);
+        } else {
+            printf("  %i\t%"PRIu32"\t%"PRIu32"\t%"PRIu32"\t%"PRIu32"\t0x%"PRIx64"\tNA\n", level,\
+                node->chrIdxStart[i], \
+                node->baseStart[i], \
+                node->chrIdxEnd[i], \
+                node->baseEnd[i], \
+                node->dataOffset[i]);
+            bwPrintIndexNode(node->x.child[i], level+1);
+        }
+    }
+}
+
+void bwPrintIndexTree(bigWigFile_t *fp) {
+    printf("\nIndex tree:\n");
+    printf("nItems:\t%"PRIu64"\n", fp->idx->nItems);
+    printf("chrIdxStart:\t%"PRIu32"\n", fp->idx->chrIdxStart);
+    printf("baseStart:\t%"PRIu32"\n", fp->idx->baseStart);
+    printf("chrIdxEnd:\t%"PRIu32"\n", fp->idx->chrIdxEnd);
+    printf("baseEnd:\t%"PRIu32"\n", fp->idx->baseEnd);
+    printf("idxSize:\t%"PRIu64"\n", fp->idx->idxSize);
+    printf("  level\tchrIdxStart\tbaseStart\tchrIdxEnd\tbaseEnd\tchild\tsize\n");
+    bwPrintIndexNode(fp->idx->root, 0);
+}
+
+int main(int argc, char *argv[]) {
+    bigWigFile_t *fp = NULL;
+    bbOverlappingEntries_t *o;
+    uint32_t i;
+    char *sql;
+    if(argc != 2) {
+        fprintf(stderr, "Usage: %s {file.bb|URL://path/file.bb}\n", argv[0]);
+        return 1;
+    }
+
+    if(bwInit(1<<17) != 0) {
+        fprintf(stderr, "Received an error in bwInit\n");
+        return 1;
+    }
+
+    assert(bwIsBigWig(argv[1], NULL) == 0);
+    assert(bbIsBigBed(argv[1], NULL) == 1);
+
+    fp = bbOpen(argv[1], NULL);
+    if(!fp) {
+        fprintf(stderr, "An error occured while opening %s\n", argv[1]);
+        return 1;
+    }
+
+    bwPrintHdr(fp);
+    bwPrintIndexTree(fp);
+
+    sql = bbGetSQL(fp);
+    if(sql) {
+        printf("SQL is: %s\n", sql);
+        free(sql);
+    }
+
+    //Presumably this is the sort of interface that's needed...
+    o = bbGetOverlappingEntries(fp, "chr1", 4450000, 4500000, 1);
+    printf("%"PRIu32" entries overlap\n", o->l);
+    for(i=0; i<o->l; i++) {
+        printf("%"PRIu32"-%"PRIu32"\t %s\n", o->start[i], o->end[i], o->str[i]);
+    }
+    if(o) bbDestroyOverlappingEntries(o);
+
+    //Ensure that we can fetch entries with no strings
+    o = bbGetOverlappingEntries(fp, "chr1", 4450000, 4500000, 0);
+    printf("%"PRIu32" entries overlap\n", o->l);
+    for(i=0; i<o->l; i++) {
+        printf("%"PRIu32"-%"PRIu32"\n", o->start[i], o->end[i]);
+    }
+    if(o) bbDestroyOverlappingEntries(o);
+
+    bwClose(fp);
+    bwCleanup();
+    return 0;
+}

data/ext/bio/bigwig/libBigWig/test/testIterator.c

@@ -0,0 +1,67 @@
+#include "bigWig.h"
+#include <stdio.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <assert.h>
+
+int main(int argc, char *argv[]) {
+    bigWigFile_t *fp = NULL;
+    uint32_t i, chunk = 0, tid, blocksPerIteration;
+    char *sql, *chrom;
+    bwOverlapIterator_t *iter;
+    if(argc != 4) {
+        fprintf(stderr, "Usage: %s {file.bb|URL://path/file.bb} chromosome blocksPerIteration\n", argv[0]);
+        return 1;
+    }
+    chrom = argv[2];
+    blocksPerIteration = strtoul(argv[3], NULL, 10);
+
+    if(bwInit(1<<17) != 0) {
+        fprintf(stderr, "Received an error in bwInit\n");
+        return 1;
+    }
+
+    if(bwIsBigWig(argv[1], NULL)) {
+        fp = bwOpen(argv[1], NULL, "r");
+    } else if(bbIsBigBed(argv[1], NULL)) {
+        fp = bbOpen(argv[1], NULL);
+    }
+
+    if(!fp) {
+        fprintf(stderr, "An error occured while opening %s\n", argv[1]);
+        return 1;
+    }
+
+    sql = bbGetSQL(fp);
+    if(sql) {
+        printf("SQL is: %s\n", sql);
+        free(sql);
+    }
+
+    //So we can get the bounds
+    tid = bwGetTid(fp, chrom);
+
+    if(fp->type == 0) {
+        iter = bwOverlappingIntervalsIterator(fp, chrom, 0, fp->cl->len[tid], blocksPerIteration);
+    } else {
+        iter = bbOverlappingEntriesIterator(fp, chrom, 0, fp->cl->len[tid], 1, blocksPerIteration);
+    }
+    while(iter->data) {
+        if(fp->type == 0) {
+            for(i=0; i<iter->intervals->l; i++) {
+                printf("chunk %"PRIu32" entry %"PRIu32" %s:%"PRIu32"-%"PRIu32" %f\n", chunk, i, chrom, iter->intervals->start[i], iter->intervals->end[i], iter->intervals->value[i]);
+            }
+        } else {
+            for(i=0; i<iter->entries->l; i++) {
+                printf("chunk %"PRIu32" entry %"PRIu32" %s:%"PRIu32"-%"PRIu32" %s\n", chunk, i, chrom, iter->entries->start[i], iter->entries->end[i], iter->entries->str[i]);
+            }
+        }
+        chunk++;
+        iter = bwIteratorNext(iter);
+    }
+    bwIteratorDestroy(iter);
+
+    bwClose(fp);
+    bwCleanup();
+    return 0;
+}