ultragrep 0.1.0 → 0.10.0

data/src/ug_lua.h

@@ -0,0 +1,10 @@
+#ifndef _UG_LUA_H
+#define _UG_LUA_H
+
+#include <lua.h>
+
+void ug_process_line(lua_State *lua, char *line, int line_len, off_t offset);
+void ug_lua_on_eof(lua_State *lua);
+lua_State *ug_lua_init(char *fname);
+
+#endif

metadata

@@ -1,67 +1,64 @@
 --- !ruby/object:Gem::Specification
 name: ultragrep
 version: !ruby/object:Gem::Version
-  version: 0.1.0
-  prerelease: 
+  version: 0.10.0
 platform: ruby
 authors:
-- John Doe
+- Ben Osheroff
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-27 00:00:00.000000000 Z
+date: 2014-08-20 00:00:00.000000000 Z
 dependencies: []
 description: 
-email: john@example.com
+email: ben@zendesk.com
 executables:
 - ultragrep
 extensions:
-- ext/ultragrep/extconf.rb
+- src/extconf.rb
 extra_rdoc_files: []
 files:
 - bin/ultragrep
-- ext/ultragrep/Makefile
-- ext/ultragrep/extconf.rb
-- ext/ultragrep/rails_req.c
-- ext/ultragrep/rails_req.h
-- ext/ultragrep/req_matcher.h
-- ext/ultragrep/request.c
-- ext/ultragrep/request.h
-- ext/ultragrep/ug_build_index.c
-- ext/ultragrep/ug_cat.c
-- ext/ultragrep/ug_guts.c
-- ext/ultragrep/ug_index.c
-- ext/ultragrep/ug_index.h
-- ext/ultragrep/work_req.c
-- ext/ultragrep/work_req.h
-- ext/ultragrep/zran.c
+- bin/ultragrep_build_indexes
 - lib/ultragrep.rb
 - lib/ultragrep/config.rb
+- lib/ultragrep/log_collector.rb
 - lib/ultragrep/version.rb
+- src/Makefile
+- src/extconf.rb
+- src/pcre.h
+- src/request.h
+- src/ug_build_index.c
+- src/ug_cat.c
+- src/ug_guts.c
+- src/ug_gzip.c
+- src/ug_gzip.h
+- src/ug_index.c
+- src/ug_index.h
+- src/ug_lua.c
+- src/ug_lua.h
 homepage: https://github.com/zendesk/ultragrep
 licenses:
 - Apache License Version 2.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Ultragrep
 test_files: []
-has_rdoc: 

data/ext/ultragrep/Makefile

@@ -1,39 +0,0 @@
-CFLAGS=-I/opt/local/include -g
-all: ug_guts ug_build_index ug_cat 
-install: all
-
-request.o: request.c request.h
-	gcc ${CFLAGS} -c request.c
-
-rails_req.o: rails_req.c rails_req.h req_matcher.h
-	gcc ${CFLAGS} -c rails_req.c
-
-work_req.o: work_req.c work_req.h req_matcher.h
-	gcc ${CFLAGS} -c work_req.c
-
-ug_guts.o: ug_guts.c req_matcher.h
-	gcc ${CFLAGS} -c ug_guts.c
-
-ug_guts: ug_guts.o rails_req.o work_req.o request.o Makefile
-	gcc ${CFLAGS} -o ug_guts request.o rails_req.o work_req.o ug_guts.o -lpcre
-
-ug_index.o: ug_index.h ug_index.c
-	gcc ${CFLAGS} -c ug_index.c
-
-ug_build_index.o: ug_build_index.c req_matcher.h ug_index.h
-	gcc ${CFLAGS} -c ug_build_index.c
-
-ug_build_index: ug_build_index.o ug_index.o rails_req.o work_req.o request.o Makefile zran.o
-	gcc ${CFLAGS} -o ug_build_index request.o rails_req.o work_req.o ug_index.o ug_build_index.o zran.o -lpcre -lz
-
-ug_cat.o: ug_cat.c ug_index.h
-	gcc ${CFLAGS} -c ug_cat.c
-
-ug_cat: ug_cat.o ug_index.o Makefile
-	gcc ${CFLAGS} -o ug_cat ug_cat.o ug_index.o
-
-zran.o: zran.c
-	gcc -g -c zran.c
-
-clean:
-	rm -rf *.o ug_guts

data/ext/ultragrep/rails_req.c

@@ -1,102 +0,0 @@
-#include <stdio.h>
-#include <time.h>
-#include "pcre.h"
-#include "request.h"
-#include "req_matcher.h"
-
-typedef struct {
-    req_matcher_t base;
-    on_req on_request;
-    on_err on_error;
-    void* arg;
-    int stop_requested;
-    int blank_lines;
-
-}rails_req_matcher_t;
-static request_t request;
-
-
-static void rails_on_request(rails_req_matcher_t* m, request_t* r) {
-    if(r && m->on_request) {
-        if(r->lines > 0) {
-            m->on_request(r, m->arg);
-        }
-        clear_request(r);
-    }
-}
-
-void rails_stop(req_matcher_t* base) {
-    rails_req_matcher_t* m = (rails_req_matcher_t*)base;
-    m->stop_requested = 1;
-}
-
-static int parse_req_time(char* line, ssize_t line_size, time_t* time) {
-    int matched = 0;
-    int ovector[30];
-    char *date_buf;
-    struct tm request_tm;
-    time_t tv;
-    const char* error;
-    int erroffset;
-    static pcre* regex = NULL;
-
-    *time = 0;
-
-    if(regex == NULL) {
-        regex = pcre_compile("^(?:Processing|Started).*(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})", 0, &error, &erroffset, NULL);
-    }
-    matched = pcre_exec(regex, NULL, line, line_size,0,0,ovector, 30);
-    if(matched > 0) {
-        pcre_get_substring(line, ovector, matched, 1, (const char **)&date_buf);
-        strptime(date_buf, "%Y-%m-%d %H:%M:%S", &request_tm);
-        free(date_buf);
-
-        *time = timegm(&request_tm);
-        return(1);
-    }
-    return(-1);
-}
-
-static int rails_process_line(req_matcher_t* base, char *line, ssize_t line_size, off_t offset) {
-    rails_req_matcher_t* m = (rails_req_matcher_t*)base;
-
-    if((m->stop_requested) || (line_size == -1)) {
-        rails_on_request(m, &request);
-        return((m->stop_requested)?STOP_SIGNAL:EOF_REACHED);
-    }
-
-    if(line_size == 1) { //blank line
-        m->blank_lines += 1;
-        return(0);
-    }
-
-    if(m->blank_lines >= 2) {
-        m->blank_lines = 0;
-        rails_on_request(m, &request);
-    }
-
-    add_to_request(&request, line, offset);
-
-    if(request.time == 0) {
-        parse_req_time(line, line_size, &(request.time));
-    }
-
-    return(0);
-}
-
-req_matcher_t* rails_req_matcher(on_req fn1, on_err fn2, void* arg) {
-    rails_req_matcher_t* m = (rails_req_matcher_t*)malloc(sizeof(rails_req_matcher_t));
-    req_matcher_t* base = (req_matcher_t*)m;
-
-    m->on_request = fn1;
-    m->on_error = fn2;
-    m->arg = arg;
-
-    m->stop_requested = 0;
-    m->blank_lines = 0;
-
-    base->process_line = &rails_process_line;
-    base->stop = &rails_stop;
-    clear_request(&request);
-    return base;
-}

data/ext/ultragrep/rails_req.h

@@ -1,6 +0,0 @@
-#ifndef __RAILS_REQ_H__
-#define __RAILS_REQ_H__
-#include "req_matcher.h"
-
-req_matcher_t* rails_req_matcher(on_req fn1, on_err fn2, void* arg);
-#endif

data/ext/ultragrep/req_matcher.h

@@ -1,17 +0,0 @@
-#ifndef __REQ_MATCHER_H__
-#define __REQ_MATCHER_H__
-#include "request.h"
-#include <sys/types.h>
-
-typedef void (*on_req)(request_t*, void* arg);
-typedef void (*on_err)(char*, ssize_t, void* arg);
-
-typedef struct req_matcher_t{
-    int (*process_line)(struct req_matcher_t* base, char* line, ssize_t line_sz, off_t offset);
-    void (*stop)(struct req_matcher_t* base);
-}req_matcher_t;
-
-#define EOF_REACHED 1
-#define STOP_SIGNAL 2
-
-#endif //__REQ_MATCHER_H__

data/ext/ultragrep/request.c

@@ -1,41 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "request.h"
-
-request_t* alloc_request() {
-    request_t* r = (request_t*)calloc(1, sizeof(request_t));
-    return(r);
-}
-
-void init_request(request_t* r) {
-    memset(r, 0, sizeof(request_t));
-}
-void clear_request(request_t* r) {
-    int i=0;
-
-    for(i = 0; i < r->lines; i++) {
-        free(r->buf[i]);
-    }
-    if(r->buf) {
-        free(r->buf);
-    }
-    if(r->session) {
-        free(r->session);
-    }
-    init_request(r);
-}
-
-void free_request(request_t* r) {
-    clear_request(r);
-    free(r);
-}
-
-void add_to_request(request_t* req, char* line, off_t offset) {
-    if ( !req->offset ) 
-        req->offset = offset;
-
-    req->buf = realloc(req->buf, sizeof(char*) * (req->lines + 1));
-    req->buf[req->lines] = line;
-    req->lines++;
-}

data/ext/ultragrep/request.h

@@ -1,22 +0,0 @@
-#ifndef __REQUEST_H__
-#define __REQUEST_H__
-
-#include <time.h>
-
-typedef struct request_t{
-    char **buf;
-    int lines;
-    time_t time;
-    char* session;
-    off_t offset;
-
-    struct request_t* next; //for linking
-    struct request_t* prev;
-}request_t;
-
-request_t* alloc_request();
-void init_request(request_t* r);
-void clear_request(request_t* r);
-void free_request(request_t* r);
-void add_to_request(request_t*, char*, off_t);
-#endif //__REQUEST_H__

data/ext/ultragrep/ug_build_index.c

@@ -1,99 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <string.h>
-#include <time.h>
-#include <unistd.h>
-#include "pcre.h"
-#include "rails_req.h"
-#include "work_req.h"
-#include "ug_index.h"
-
-#define USAGE "Usage: ug_build_index (work|app) file\n"
-
-
-// index file format
-// [64bit,64bit] -- timestamp, file offset 
-// [32bit, Nbytes] -- extra data
-
-void handle_request(request_t* req, build_idx_context_t* cxt) {
-    time_t floored_time;
-    floored_time = req->time - (req->time % INDEX_EVERY);
-    if ( !cxt->last_index_time || floored_time > cxt->last_index_time ) {
-        ug_write_index(cxt->index, floored_time, req->offset, NULL, 0);
-        cxt->last_index_time = floored_time;
-    }
-}
-
-int main(int argc, char **argv)
-{
-    build_idx_context_t *cxt;
-    char *line = NULL, *index_fname = NULL, *dir;
-    ssize_t line_size, allocated;
-
-
-    if ( argc < 3 ) {
-        fprintf(stderr, USAGE);
-        exit(1);
-    }
-
-    cxt = malloc(sizeof(build_idx_context_t));
-    memset(cxt, 0, sizeof(build_idx_context_t));
-
-    if(strcmp(argv[1],"work") == 0)
-    {
-        cxt->m = work_req_matcher(&handle_request, NULL, cxt);
-    }
-    else if(strcmp(argv[1], "app") == 0)
-    {
-        cxt->m = rails_req_matcher(&handle_request, NULL, cxt);
-    }
-    else
-    {
-        fprintf(stderr, USAGE);
-        exit(1);
-    }
-
-    cxt->log = fopen(argv[2], "r");
-    if ( !cxt->log ) { 
-      perror("Couldn't open log file");
-      exit(1);
-    }
-
-    index_fname = ug_get_index_fname(argv[2]);
-
-    cxt->index = fopen(index_fname, "r+");
-    if ( cxt->index ) { 
-        struct ug_index idx;
-        ug_get_last_index_entry(cxt->index, &idx);
-        fseeko(cxt->log, idx.offset, SEEK_SET);
-    } else {
-        cxt->index = fopen(index_fname, "w+");
-    }
-
-    if ( !cxt->index ) { 
-      perror("Couldn't open index file");
-      exit(1);
-    }
-
-    if ( strcmp(argv[2] + (strlen(argv[2]) - 3), ".gz") == 0 ) {
-      build_gz_index(cxt);
-    } else {
-      cxt->data_size = 0;
-
-      while(1) {
-          int ret;
-          line_size = getline(&line, &allocated, cxt->log);
-          ret = cxt->m->process_line(cxt->m, line, line_size, ftello(cxt->log) - line_size);
-          if(ret == EOF_REACHED || ret == STOP_SIGNAL) {
-              break;
-          }
-          line = NULL;
-      }
-    }
-
-    fclose(cxt->index);
-    fclose(cxt->log);
-}
-
-

data/ext/ultragrep/ug_cat.c

@@ -1,46 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <libgen.h>
-#include "ug_index.h"
-
-/* 
- * ug_cat -- given a log file and (possibly) a file + (timestamp -> offset) index, cat the file starting 
- *           from about that timestamp 
- */ 
-
-#define USAGE "Usage: ug_cat file timestamp\n"
-
-int main(int argc, char **argv)
-{
-    int nread;
-    FILE *log;
-    FILE *index;
-    char *index_fname, buf[4096];
-
-    if ( argc < 3 ) {
-        fprintf(stderr, USAGE);
-        exit(1);
-    }
-
-    log = fopen(argv[1], "r");
-    if ( !log ) { 
-      perror("Couldn't open log file");
-      exit(1);
-    }
-
-    index_fname = ug_get_index_fname(argv[1]);
-
-    index = fopen(index_fname, "r");
-    if ( index ) { 
-        ug_seek_to_timestamp(log, index, atol(argv[2]), NULL);
-    }
-
-    while ( nread = fread(buf, 1, 4096, log) )  {
-        fwrite(buf, 1, nread, stdout);
-    }
-
-    fclose(log);
-}
-
-