ultragrep 0.1.0

data/bin/ultragrep

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
+require "ultragrep"
+
+Thread.abort_on_exception = true
+Ultragrep::ultragrep(Ultragrep::parse_args(ARGV))

data/ext/ultragrep/Makefile

@@ -0,0 +1,39 @@
+CFLAGS=-I/opt/local/include -g
+all: ug_guts ug_build_index ug_cat 
+install: all
+
+request.o: request.c request.h
+	gcc ${CFLAGS} -c request.c
+
+rails_req.o: rails_req.c rails_req.h req_matcher.h
+	gcc ${CFLAGS} -c rails_req.c
+
+work_req.o: work_req.c work_req.h req_matcher.h
+	gcc ${CFLAGS} -c work_req.c
+
+ug_guts.o: ug_guts.c req_matcher.h
+	gcc ${CFLAGS} -c ug_guts.c
+
+ug_guts: ug_guts.o rails_req.o work_req.o request.o Makefile
+	gcc ${CFLAGS} -o ug_guts request.o rails_req.o work_req.o ug_guts.o -lpcre
+
+ug_index.o: ug_index.h ug_index.c
+	gcc ${CFLAGS} -c ug_index.c
+
+ug_build_index.o: ug_build_index.c req_matcher.h ug_index.h
+	gcc ${CFLAGS} -c ug_build_index.c
+
+ug_build_index: ug_build_index.o ug_index.o rails_req.o work_req.o request.o Makefile zran.o
+	gcc ${CFLAGS} -o ug_build_index request.o rails_req.o work_req.o ug_index.o ug_build_index.o zran.o -lpcre -lz
+
+ug_cat.o: ug_cat.c ug_index.h
+	gcc ${CFLAGS} -c ug_cat.c
+
+ug_cat: ug_cat.o ug_index.o Makefile
+	gcc ${CFLAGS} -o ug_cat ug_cat.o ug_index.o
+
+zran.o: zran.c
+	gcc -g -c zran.c
+
+clean:
+	rm -rf *.o ug_guts

data/ext/ultragrep/extconf.rb

@@ -0,0 +1 @@
+# Do nothing, we already have a complete Makefile!

data/ext/ultragrep/rails_req.c

@@ -0,0 +1,102 @@
+#include <stdio.h>
+#include <time.h>
+#include "pcre.h"
+#include "request.h"
+#include "req_matcher.h"
+
+typedef struct {
+    req_matcher_t base;
+    on_req on_request;
+    on_err on_error;
+    void* arg;
+    int stop_requested;
+    int blank_lines;
+
+}rails_req_matcher_t;
+static request_t request;
+
+
+static void rails_on_request(rails_req_matcher_t* m, request_t* r) {
+    if(r && m->on_request) {
+        if(r->lines > 0) {
+            m->on_request(r, m->arg);
+        }
+        clear_request(r);
+    }
+}
+
+void rails_stop(req_matcher_t* base) {
+    rails_req_matcher_t* m = (rails_req_matcher_t*)base;
+    m->stop_requested = 1;
+}
+
+static int parse_req_time(char* line, ssize_t line_size, time_t* time) {
+    int matched = 0;
+    int ovector[30];
+    char *date_buf;
+    struct tm request_tm;
+    time_t tv;
+    const char* error;
+    int erroffset;
+    static pcre* regex = NULL;
+
+    *time = 0;
+
+    if(regex == NULL) {
+        regex = pcre_compile("^(?:Processing|Started).*(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})", 0, &error, &erroffset, NULL);
+    }
+    matched = pcre_exec(regex, NULL, line, line_size,0,0,ovector, 30);
+    if(matched > 0) {
+        pcre_get_substring(line, ovector, matched, 1, (const char **)&date_buf);
+        strptime(date_buf, "%Y-%m-%d %H:%M:%S", &request_tm);
+        free(date_buf);
+
+        *time = timegm(&request_tm);
+        return(1);
+    }
+    return(-1);
+}
+
+static int rails_process_line(req_matcher_t* base, char *line, ssize_t line_size, off_t offset) {
+    rails_req_matcher_t* m = (rails_req_matcher_t*)base;
+
+    if((m->stop_requested) || (line_size == -1)) {
+        rails_on_request(m, &request);
+        return((m->stop_requested)?STOP_SIGNAL:EOF_REACHED);
+    }
+
+    if(line_size == 1) { //blank line
+        m->blank_lines += 1;
+        return(0);
+    }
+
+    if(m->blank_lines >= 2) {
+        m->blank_lines = 0;
+        rails_on_request(m, &request);
+    }
+
+    add_to_request(&request, line, offset);
+
+    if(request.time == 0) {
+        parse_req_time(line, line_size, &(request.time));
+    }
+
+    return(0);
+}
+
+req_matcher_t* rails_req_matcher(on_req fn1, on_err fn2, void* arg) {
+    rails_req_matcher_t* m = (rails_req_matcher_t*)malloc(sizeof(rails_req_matcher_t));
+    req_matcher_t* base = (req_matcher_t*)m;
+
+    m->on_request = fn1;
+    m->on_error = fn2;
+    m->arg = arg;
+
+    m->stop_requested = 0;
+    m->blank_lines = 0;
+
+    base->process_line = &rails_process_line;
+    base->stop = &rails_stop;
+    clear_request(&request);
+    return base;
+}

data/ext/ultragrep/rails_req.h

@@ -0,0 +1,6 @@
+#ifndef __RAILS_REQ_H__
+#define __RAILS_REQ_H__
+#include "req_matcher.h"
+
+req_matcher_t* rails_req_matcher(on_req fn1, on_err fn2, void* arg);
+#endif

data/ext/ultragrep/req_matcher.h

@@ -0,0 +1,17 @@
+#ifndef __REQ_MATCHER_H__
+#define __REQ_MATCHER_H__
+#include "request.h"
+#include <sys/types.h>
+
+typedef void (*on_req)(request_t*, void* arg);
+typedef void (*on_err)(char*, ssize_t, void* arg);
+
+typedef struct req_matcher_t{
+    int (*process_line)(struct req_matcher_t* base, char* line, ssize_t line_sz, off_t offset);
+    void (*stop)(struct req_matcher_t* base);
+}req_matcher_t;
+
+#define EOF_REACHED 1
+#define STOP_SIGNAL 2
+
+#endif //__REQ_MATCHER_H__

data/ext/ultragrep/request.c

@@ -0,0 +1,41 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "request.h"
+
+request_t* alloc_request() {
+    request_t* r = (request_t*)calloc(1, sizeof(request_t));
+    return(r);
+}
+
+void init_request(request_t* r) {
+    memset(r, 0, sizeof(request_t));
+}
+void clear_request(request_t* r) {
+    int i=0;
+
+    for(i = 0; i < r->lines; i++) {
+        free(r->buf[i]);
+    }
+    if(r->buf) {
+        free(r->buf);
+    }
+    if(r->session) {
+        free(r->session);
+    }
+    init_request(r);
+}
+
+void free_request(request_t* r) {
+    clear_request(r);
+    free(r);
+}
+
+void add_to_request(request_t* req, char* line, off_t offset) {
+    if ( !req->offset ) 
+        req->offset = offset;
+
+    req->buf = realloc(req->buf, sizeof(char*) * (req->lines + 1));
+    req->buf[req->lines] = line;
+    req->lines++;
+}

data/ext/ultragrep/request.h

@@ -0,0 +1,22 @@
+#ifndef __REQUEST_H__
+#define __REQUEST_H__
+
+#include <time.h>
+
+typedef struct request_t{
+    char **buf;
+    int lines;
+    time_t time;
+    char* session;
+    off_t offset;
+
+    struct request_t* next; //for linking
+    struct request_t* prev;
+}request_t;
+
+request_t* alloc_request();
+void init_request(request_t* r);
+void clear_request(request_t* r);
+void free_request(request_t* r);
+void add_to_request(request_t*, char*, off_t);
+#endif //__REQUEST_H__

data/ext/ultragrep/ug_build_index.c

@@ -0,0 +1,99 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include "pcre.h"
+#include "rails_req.h"
+#include "work_req.h"
+#include "ug_index.h"
+
+#define USAGE "Usage: ug_build_index (work|app) file\n"
+
+
+// index file format
+// [64bit,64bit] -- timestamp, file offset 
+// [32bit, Nbytes] -- extra data
+
+void handle_request(request_t* req, build_idx_context_t* cxt) {
+    time_t floored_time;
+    floored_time = req->time - (req->time % INDEX_EVERY);
+    if ( !cxt->last_index_time || floored_time > cxt->last_index_time ) {
+        ug_write_index(cxt->index, floored_time, req->offset, NULL, 0);
+        cxt->last_index_time = floored_time;
+    }
+}
+
+int main(int argc, char **argv)
+{
+    build_idx_context_t *cxt;
+    char *line = NULL, *index_fname = NULL, *dir;
+    ssize_t line_size, allocated;
+
+
+    if ( argc < 3 ) {
+        fprintf(stderr, USAGE);
+        exit(1);
+    }
+
+    cxt = malloc(sizeof(build_idx_context_t));
+    memset(cxt, 0, sizeof(build_idx_context_t));
+
+    if(strcmp(argv[1],"work") == 0)
+    {
+        cxt->m = work_req_matcher(&handle_request, NULL, cxt);
+    }
+    else if(strcmp(argv[1], "app") == 0)
+    {
+        cxt->m = rails_req_matcher(&handle_request, NULL, cxt);
+    }
+    else
+    {
+        fprintf(stderr, USAGE);
+        exit(1);
+    }
+
+    cxt->log = fopen(argv[2], "r");
+    if ( !cxt->log ) { 
+      perror("Couldn't open log file");
+      exit(1);
+    }
+
+    index_fname = ug_get_index_fname(argv[2]);
+
+    cxt->index = fopen(index_fname, "r+");
+    if ( cxt->index ) { 
+        struct ug_index idx;
+        ug_get_last_index_entry(cxt->index, &idx);
+        fseeko(cxt->log, idx.offset, SEEK_SET);
+    } else {
+        cxt->index = fopen(index_fname, "w+");
+    }
+
+    if ( !cxt->index ) { 
+      perror("Couldn't open index file");
+      exit(1);
+    }
+
+    if ( strcmp(argv[2] + (strlen(argv[2]) - 3), ".gz") == 0 ) {
+      build_gz_index(cxt);
+    } else {
+      cxt->data_size = 0;
+
+      while(1) {
+          int ret;
+          line_size = getline(&line, &allocated, cxt->log);
+          ret = cxt->m->process_line(cxt->m, line, line_size, ftello(cxt->log) - line_size);
+          if(ret == EOF_REACHED || ret == STOP_SIGNAL) {
+              break;
+          }
+          line = NULL;
+      }
+    }
+
+    fclose(cxt->index);
+    fclose(cxt->log);
+}
+
+

data/ext/ultragrep/ug_cat.c

@@ -0,0 +1,46 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <libgen.h>
+#include "ug_index.h"
+
+/* 
+ * ug_cat -- given a log file and (possibly) a file + (timestamp -> offset) index, cat the file starting 
+ *           from about that timestamp 
+ */ 
+
+#define USAGE "Usage: ug_cat file timestamp\n"
+
+int main(int argc, char **argv)
+{
+    int nread;
+    FILE *log;
+    FILE *index;
+    char *index_fname, buf[4096];
+
+    if ( argc < 3 ) {
+        fprintf(stderr, USAGE);
+        exit(1);
+    }
+
+    log = fopen(argv[1], "r");
+    if ( !log ) { 
+      perror("Couldn't open log file");
+      exit(1);
+    }
+
+    index_fname = ug_get_index_fname(argv[1]);
+
+    index = fopen(index_fname, "r");
+    if ( index ) { 
+        ug_seek_to_timestamp(log, index, atol(argv[2]), NULL);
+    }
+
+    while ( nread = fread(buf, 1, 4096, log) )  {
+        fwrite(buf, 1, nread, stdout);
+    }
+
+    fclose(log);
+}
+
+

data/ext/ultragrep/ug_guts.c

@@ -0,0 +1,138 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <string.h>
+#include <time.h>
+#include "pcre.h"
+#include "req_matcher.h"
+#include "rails_req.h"
+#include "work_req.h"
+
+
+typedef struct {
+    time_t start_time;
+    time_t end_time;
+    int num_regexps;
+    pcre **regexps;
+    req_matcher_t* m;
+}context_t;
+
+
+int check_request(int lines, char **request, time_t request_time, pcre **regexps, int num_regexps)
+{
+    int *matches, i, j, matched;
+
+    matches = malloc(sizeof(int) * num_regexps);
+    memset(matches, 0, (sizeof(int) * num_regexps));
+
+    for(i=0; i < lines; i++) {
+        for(j=0; j < num_regexps; j++) {
+            int ovector[30];
+            if ( matches[j] ) continue;
+
+            matched = pcre_exec(regexps[j], NULL, request[i], strlen(request[i]), 0, 0, ovector, 30);
+            if ( matched > 0 )
+                matches[j] = 1;
+        }
+    }
+
+    matched = 1;
+    for (j=0; j < num_regexps; j++) {
+        matched &= matches[j];
+    }
+
+    free(matches);
+    return(matched);
+}
+
+void print_request(int request_lines, char **request)
+{
+    int i, j;
+    putchar('\n');
+
+    for(i=0; i < request_lines; i++)
+        printf("%s", request[i]);
+
+    for(j=0; j < strlen(request[request_lines - 1]) && j < 80; j++ )
+        putchar('-');
+
+    putchar('\n');
+    fflush(stdout);
+}
+
+void handle_request(request_t* req, void* cxt_arg) {
+    static int time = 0;
+    context_t* cxt = (context_t*)cxt_arg;
+    if( (req->time > cxt->start_time &&
+            check_request(req->lines,  req->buf, req->time, cxt->regexps, cxt->num_regexps))) {
+    if(req->time != 0) {
+            printf("@@%lu\n", req->time);
+    }
+
+        print_request(req->lines, req->buf);
+    }
+    if(req->time > time) {
+    time = req->time;
+        printf("@@%lu\n", time);
+    }
+    if(req->time > cxt->end_time) {
+        cxt->m->stop(cxt->m);
+    }
+}
+int main(int argc, char **argv)
+{
+    int i;
+    context_t *cxt;
+    const char *error;
+    int erroffset;
+    char *line = NULL;
+    ssize_t line_size, allocated;
+
+
+    if ( argc < 5 ) {
+        fprintf(stderr, "Usage: ug_guts (work|app) start_time end_time regexps [... regexps]\n");
+        exit(1);
+    }
+
+    cxt = malloc(sizeof(context_t));
+
+    if(strcmp(argv[1],"work") == 0)
+    {
+        cxt->m = work_req_matcher(&handle_request, NULL, cxt);
+    }
+    else if(strcmp(argv[1], "app") == 0)
+    {
+        cxt->m = rails_req_matcher(&handle_request, NULL, cxt);
+    }
+    else
+    {
+        fprintf(stderr, "Usage: ug_guts (work|app) start_time end_time regexps [... regexps]\n");
+        exit(1);
+    }
+
+    cxt->start_time = atol(argv[2]);
+    cxt->end_time = atol(argv[3]);
+
+    cxt->num_regexps = argc - 4;
+    cxt->regexps = malloc(sizeof(pcre *) * cxt->num_regexps);
+
+    for ( i = 4; i < argc; i++) {
+        cxt->regexps[i-4] = pcre_compile(argv[i], 0, &error, &erroffset, NULL);
+        if ( error ) {
+            fprintf(stderr, "Error compiling regexp \"%s\": %s\n", argv[i], error);
+            exit;
+        }
+    }
+
+
+    while(1) {
+        int ret;
+        line_size = getline(&line, &allocated, stdin);
+        ret = cxt->m->process_line(cxt->m, line, line_size, 0);
+        if(ret == EOF_REACHED || ret == STOP_SIGNAL) {
+            break;
+        }
+        line = NULL;
+    }
+}
+

data/ext/ultragrep/ug_index.c

@@ -0,0 +1,83 @@
+#include <stdlib.h>
+#include <string.h>
+#include <libgen.h>
+#include "ug_index.h"
+
+int ug_write_index(FILE *file, uint64_t time, uint64_t offset, char *data, uint32_t data_size)
+{
+  fwrite(&time, 8, 1, file);
+  fwrite(&offset, 8, 1, file);
+  fwrite(&data_size, 4, 1, file);
+
+  if ( data_size ) 
+    fwrite(data, 1, data_size, file);
+}
+
+int ug_read_index_entry(FILE *file, struct ug_index *idx, int read_data)
+{
+  int nread;
+  nread = fread(&(idx->time), 8, 1, file);
+  if ( !nread ) 
+    return 0;
+
+  nread = fread(&(idx->offset), 8, 1, file);
+  nread = fread(&(idx->data_size), 4, 1, file);
+  if ( idx->data_size ) {
+    if ( read_data ) {
+      idx->data = malloc(idx->data_size);
+      nread = fread(idx->data, 1, idx->data_size, file);
+    } else {
+      fseek(file, idx->data_size, SEEK_CUR);
+    }
+  }
+
+  return 1;
+}
+
+int ug_get_last_index_entry(FILE *file, struct ug_index *idx) {
+  while (ug_read_index_entry(file, idx, 0));
+}
+
+void ug_seek_to_timestamp(FILE *flog, FILE *findex, uint64_t time, struct ug_index *param_idx) 
+{
+  struct ug_index idx, prev;
+  off_t last_offset = 0;
+
+  memset(&prev, 0, sizeof(struct ug_index));
+
+  for(;;) { 
+    if ( !ug_read_index_entry(findex, &idx, 0) ) {
+      memcpy(&prev, &idx, sizeof(struct ug_index));
+      break;
+    }
+
+    if ( idx.time > time )
+      break;
+
+    memcpy(&prev, &idx, sizeof(struct ug_index));
+  } 
+
+  if ( prev.offset ) {
+      fseek(flog, prev.offset, SEEK_SET);
+      if ( param_idx ) {
+        memcpy(param_idx, &prev, sizeof(struct ug_index));
+      }
+  }
+}
+
+/* returns malloc'ed memory. */
+char *ug_get_index_fname(char *log_fname) 
+{
+  char *dir, *index_fname;
+
+  dir = strdup(log_fname);
+  dir = dirname(dir);
+
+  index_fname = malloc(strlen(dir) + strlen(basename(log_fname)) + strlen("/..idx") + 1);
+
+  sprintf(index_fname, "%s/.%s.idx", dir, basename(log_fname));
+  free(dir);
+  return index_fname;
+}
+
+

data/ext/ultragrep/ug_index.h

@@ -0,0 +1,27 @@
+#include <stdint.h>
+#include <stdio.h>
+#include "req_matcher.h"
+
+#define INDEX_EVERY 10
+
+struct ug_index {
+  uint64_t time;
+  uint64_t offset;
+  uint32_t data_size;
+  char *data;
+};
+
+typedef struct {
+    time_t last_index_time;
+    FILE *log;
+    FILE *index;
+    uint32_t data_size;
+    req_matcher_t* m;
+    unsigned char data[32768];
+} build_idx_context_t;
+
+int ug_write_index(FILE *file, uint64_t time, uint64_t offset, char *data, uint32_t data_size);
+int ug_get_last_index_entry(FILE *file, struct ug_index *idx); 
+void ug_seek_to_timestamp(FILE *log, FILE *idx, uint64_t time, struct ug_index *param_idx);
+char *ug_get_index_fname(char *log_fname);
+