ultragrep 0.1.0 → 0.10.0

data/src/request.h

@@ -0,0 +1,13 @@
+#ifndef __REQUEST_H__
+#define __REQUEST_H__
+
+#include <time.h>
+
+typedef struct request_t {
+    char *buf;
+    off_t offset;
+    time_t time;
+} request_t;
+
+void handle_request(request_t * req);
+#endif                          //__REQUEST_H__

data/src/ug_build_index.c

@@ -0,0 +1,109 @@
+// ex: set softtabstop=4 shiftwidth=4 tabstop=4 expandtab:
+#include <stdio.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <string.h>
+#include <errno.h>
+#include <time.h>
+#include <unistd.h>
+#include "pcre.h"
+#include "request.h"
+#include "ug_index.h"
+#include "ug_lua.h"
+#include "ug_gzip.h"
+
+#define USAGE "Usage: ug_build_index process.lua file\n"
+
+// index file format
+// [64bit,64bit] -- timestamp, file offset 
+// [32bit, Nbytes] -- extra data
+
+static build_idx_context_t ctx;
+
+
+void handle_request(request_t *req)
+{
+    time_t floored_time;
+    floored_time = req->time - (req->time % INDEX_EVERY);
+    if (!ctx.last_index_time || floored_time > ctx.last_index_time) {
+        ug_write_index(ctx.findex, floored_time, req->offset);
+        ctx.last_index_time = floored_time;
+    }
+}
+
+void open_indexes(char *log_fname)
+{
+    char *index_fname, *gz_index_fname;
+
+    index_fname = ug_get_index_fname(log_fname, "idx");
+
+    if (strcmp(log_fname + (strlen(log_fname) - 3), ".gz") == 0) {
+        gz_index_fname = ug_get_index_fname(log_fname, "gzidx");
+        /* we don't do incremental index building in gzipped files -- we just truncate and 
+         * build over*/
+        ctx.findex = fopen(index_fname, "w+");
+        ctx.fgzindex = fopen(gz_index_fname, "w+");
+
+        if (!ctx.findex || !ctx.fgzindex) {
+            fprintf(stderr, "Couldn't open index files '%s','%s': %s\n", index_fname, gz_index_fname, strerror(errno));
+            exit(1);
+        }
+    } else {
+        ctx.findex = fopen(index_fname, "r+");
+        if (ctx.findex) {
+            /* seek in the log, (and the index, with get_offset_for_timestamp()) to the 
+             * last timestamp we indexed */
+            fseeko(ctx.flog, ug_get_offset_for_timestamp(ctx.findex, -1), SEEK_SET);
+        } else {
+            ctx.findex = fopen(index_fname, "w+");
+        }
+        if (!ctx.findex) {
+            fprintf(stderr, "Couldn't open index file '%s': %s\n", index_fname, strerror(errno));
+            exit(1);
+        }
+    }
+}
+
+int main(int argc, char **argv)
+{
+    char *line = NULL, *lua_fname, *log_fname;
+    ssize_t line_size;
+    size_t allocated;
+
+    if (argc < 3) {
+        fprintf(stderr, USAGE);
+        exit(1);
+    }
+
+    lua_fname = argv[1];
+    log_fname = argv[2];
+
+    bzero(&ctx, sizeof(build_idx_context_t));
+
+    ctx.lua = ug_lua_init(lua_fname);
+
+    ctx.flog = fopen(log_fname, "r");
+    if (!ctx.flog) {
+        perror("Couldn't open log file");
+        exit(1);
+    }
+
+    open_indexes(log_fname);
+
+    if (strcmp(log_fname + (strlen(log_fname) - 3), ".gz") == 0) {
+        build_gz_index(&ctx);
+    } else {
+        while (1) {
+            off_t offset;
+            offset = ftello(ctx.flog);
+            line_size = getline(&line, &allocated, ctx.flog);
+
+            if ( line_size < 0 )
+                break;
+
+            ug_process_line(ctx.lua, line, line_size, offset);
+        }
+    }
+    ug_lua_on_eof(ctx.lua);
+    exit(0);
+}

data/src/ug_cat.c

@@ -0,0 +1,188 @@
+// ex: set softtabstop=4 shiftwidth=4 tabstop=4 expandtab:
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <libgen.h>
+#include "ug_index.h"
+#include "ug_gzip.h"
+#include "zlib.h"
+
+/* target_offset is the offset in the uncompressed stream we're looking for. */
+void fill_gz_info(off_t target_offset, FILE * gz_index, unsigned char *dict_data, off_t * compressed_offset)
+{
+    off_t uncompressed_offset = 0;
+
+    for (;;) {
+        if (!fread(&uncompressed_offset, sizeof(off_t), 1, gz_index))
+            break;
+
+        if (uncompressed_offset > target_offset) {
+            return;
+        }
+
+        if (!fread(compressed_offset, sizeof(off_t), 1, gz_index))
+            break;
+
+        if (!fread(dict_data, WINSIZE, 1, gz_index))
+            break;
+    }
+    return;
+}
+
+/* Use the index to read len bytes from offset into buf, return bytes read or
+   negative for error (Z_DATA_ERROR or Z_MEM_ERROR).  If data is requested past
+   the end of the uncompressed data, then extract() will return a value less
+   than len, indicating how much as actually read into buf.  This function
+   should not return a data error unless the file was modified since the index
+   was generated.  extract() may also return Z_ERRNO if there is an error on
+   reading or seeking the input file. */
+int ug_gzip_cat(FILE * in, uint64_t time, FILE * offset_index, FILE * gz_index)
+{
+    int ret, bits;
+    off_t uncompressed_offset, compressed_offset;
+    z_stream strm;
+    unsigned char input[CHUNK];
+    unsigned char output[WINSIZE], dict[WINSIZE];
+
+    /* initialize file and inflate state to start there */
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    strm.avail_in = 0;
+    strm.next_in = Z_NULL;
+
+
+    bzero(dict, WINSIZE);
+
+    if (gz_index && offset_index) {
+        uncompressed_offset = ug_get_offset_for_timestamp(offset_index, time);
+        fill_gz_info(uncompressed_offset, gz_index, dict, &compressed_offset);
+
+        bits = compressed_offset >> 56;
+        compressed_offset = (compressed_offset & 0x00FFFFFFFFFFFFFF) - (bits ? 1 : 0);
+
+        ret = inflateInit2(&strm, -15);     /* raw inflate */
+        if (ret != Z_OK)
+            return ret;
+
+        ret = fseeko(in, compressed_offset, SEEK_SET);
+
+        if (ret != Z_OK)
+            return ret;
+    } else {
+        compressed_offset = bits = 0;
+        strm.avail_in = fread(input, 1, CHUNK, in);
+        strm.next_in = input;
+ 
+        ret = inflateInit2(&strm, 47);
+    }
+
+
+    if (ret == -1)
+        goto extract_ret;
+    if (bits) {
+        ret = getc(in);
+        if (ret == -1) {
+            ret = ferror(in) ? Z_ERRNO : Z_DATA_ERROR;
+            goto extract_ret;
+        }
+        (void) inflatePrime(&strm, bits, ret >> (8 - bits));
+    }
+
+    if (compressed_offset > 0)
+        inflateSetDictionary(&strm, dict, WINSIZE);
+
+    for (;;) {
+        strm.avail_out = WINSIZE;
+        strm.next_out = output;
+
+        if (!strm.avail_in) {
+            strm.avail_in = fread(input, 1, CHUNK, in);
+            strm.next_in = input;
+        }
+
+        if (ferror(in)) {
+            ret = Z_ERRNO;
+            goto extract_ret;
+        }
+
+        if (strm.avail_in == 0) {
+            ret = Z_DATA_ERROR;
+            goto extract_ret;
+        }
+
+        ret = inflate(&strm, Z_NO_FLUSH);       /* normal inflate */
+
+        if (ret == Z_NEED_DICT)
+            ret = Z_DATA_ERROR;
+        if (ret == Z_MEM_ERROR || ret == Z_DATA_ERROR)
+            goto extract_ret;
+
+        fwrite(output, WINSIZE - strm.avail_out, 1, stdout);
+
+        /* if reach end of stream, then don't keep trying to get more */
+        if (ret == Z_STREAM_END)
+            break;
+    }
+
+    /* clean up and return bytes read or error */
+  extract_ret:
+    (void) inflateEnd(&strm);
+    return ret;
+}
+/* 
+ * ug_cat -- given a log file and (possibly) a file + (timestamp -> offset) index, cat the file starting 
+ *           from about that timestamp 
+ */
+
+#define USAGE "Usage: ug_cat file timestamp\n"
+
+int main(int argc, char **argv)
+{
+    int nread;
+    FILE *log;
+    FILE *index;
+    char *log_fname, *index_fname, buf[4096];
+
+    if (argc < 3) {
+        fprintf(stderr, USAGE);
+        exit(1);
+    }
+
+    log_fname = argv[1];
+
+    log = fopen(log_fname, "r");
+    if (!log) {
+        perror("Couldn't open log file");
+        exit(1);
+    }
+
+    index_fname = ug_get_index_fname(log_fname, "idx");
+
+    index = fopen(index_fname, "r");
+    if (strcmp(log_fname + (strlen(log_fname) - 3), ".gz") == 0) {
+        char *gzidx_fname;
+        FILE *gzidx;
+
+        if (index) {
+            gzidx_fname = ug_get_index_fname(log_fname, "gzidx");
+            gzidx = fopen(gzidx_fname, "r");
+            if (!gzidx) {
+                perror("error opening gzidx component");
+                exit(1);
+            }
+            ug_gzip_cat(log, atol(argv[2]), index, gzidx);
+
+        } else {
+            ug_gzip_cat(log, atol(argv[2]), NULL, NULL);
+
+        }
+    } else {
+        if (index)
+            fseeko(log, ug_get_offset_for_timestamp(index, atol(argv[2])), SEEK_SET);
+
+        while ((nread = fread(buf, 1, 4096, log)))
+            fwrite(buf, 1, nread, stdout);
+    }
+}

data/src/ug_guts.c

@@ -0,0 +1,199 @@
+// ex: set softtabstop=4 shiftwidth=4 tabstop=4 expandtab:
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <lua.h>
+#include "pcre.h"
+#include "request.h"
+#include "ug_lua.h"
+
+struct ug_regexp {
+  int invert;
+  pcre *re;
+};
+
+typedef struct {
+    time_t start_time;
+    time_t end_time;
+    int num_regexps;
+    struct ug_regexp *regexps;
+    char *lua_file;
+    char *in_file;
+} context_t;
+
+static context_t ctx;
+
+static const char* commandparams="l:s:e:k:f:";
+static const char* usage ="Usage: ug_guts [-f input] -l file.lua -s start_time -e end_time regexps [... regexps]\n\n";
+
+int parse_args(int argc, char **argv)
+{
+    extern char *optarg;
+    extern int optind;
+    const char *error;
+    int erroffset, optValue=0, retValue=1, i;
+    ctx.start_time = -1;
+    ctx.end_time = -1;
+    ctx.lua_file = NULL;
+
+    while ((optValue = getopt(argc, argv, commandparams))!= -1) {
+        switch (optValue) {
+            case 'f':
+                ctx.in_file = strdup(optarg);
+                break;
+            case 'l':
+                ctx.lua_file = strdup(optarg);
+                break;
+            case 's':
+                ctx.start_time = atol(optarg);
+                break;
+            case 'e':
+                ctx.end_time = atol(optarg);
+                break;
+            case '?':
+                return(-1);
+                break;
+            case -1:    //Options exhausted
+                break;
+            default:
+                return(-1);
+            }
+    }
+    if ( ctx.lua_file == NULL ||  ctx.start_time < 0 || ctx.end_time < 0 ) {	// mandatory fields
+        return(-1);
+    }
+    else if ((optind + 1 ) > argc) { // Need at least one argument after options
+        return(-1);
+    }
+
+    if (optind < argc) {	// regexps follow after command-line options
+        ctx.num_regexps = argc - optind;
+        ctx.regexps = malloc(sizeof(struct ug_regexp) * ctx.num_regexps);
+        bzero(ctx.regexps, sizeof(struct ug_regexp) * ctx.num_regexps);
+
+        for (i=0; optind < argc; ++optind, i++) {
+            char *p = argv[optind];
+            if ( p[0] == '!' || p[0] == '+' ) {
+                ctx.regexps[i].invert = p[0] == '!';
+                p++;
+            }
+
+            ctx.regexps[i].re = pcre_compile(p, 0, &error, &erroffset, NULL);
+            if (error) {
+                fprintf(stderr, "Error compiling regexp \"%s\": %s\n", argv[optind], error);
+                exit(1);
+            }
+         }
+    }
+    return retValue;
+}
+
+int check_request(char *request, struct ug_regexp *regexps, int num_regexps)
+{
+  int j, matched, ovector[30];
+
+  for (j = 0; j < num_regexps; j++) {
+    matched = pcre_exec(regexps[j].re, NULL, request, strlen(request), 0, 0, ovector, 30);
+    if ( matched < 0 && !regexps[j].invert )
+        return 0;
+    else if ( matched >= 0 && regexps[j].invert )
+        return 0;
+  }
+
+  return 1;
+}
+
+void print_request(char *request)
+{
+    int i, last_line_len = 0;
+    char *p;
+
+    printf("%s", request);
+    p = request + (strlen(request) - 1);
+
+    /* skip trailing newlines */
+    while ( p > request && (*p == '\n') )
+      p--;
+
+    while ( p > request && (*p != '\n') ) {
+      p--;
+      last_line_len++;
+    }
+
+    for (i = 0; i < (last_line_len - 1) && i < 80; i++)
+        putchar('-');
+
+    putchar('\n');
+    fflush(stdout);
+}
+
+
+void handle_request(request_t * req)
+{
+    static time_t time = 0;
+
+    if (!req->time)
+      req->time = time;
+
+    if ((req->time >= ctx.start_time
+          && req->time <= ctx.end_time
+          && check_request(req->buf, ctx.regexps, ctx.num_regexps))) {
+        if (req->time != 0) {
+            printf("@@%lu\n", req->time);
+        }
+        print_request(req->buf);
+    }
+    /* print a time-marker every second -- allows collections of logs with one sparse
+       log to proceed */
+    if (req->time > time) {
+        time = req->time;
+        printf("@@%lu\n", time);
+    }
+}
+
+
+
+int main(int argc, char **argv)
+{
+    lua_State *lua;
+    ssize_t line_size;
+    FILE *file = NULL;
+    char *line = NULL;
+    size_t allocated = 0, offset = 0;
+    if (argc < 5) {
+        fprintf(stderr, "%s", usage);
+        exit(1);
+    }
+
+    bzero(&ctx, sizeof(context_t));
+    if ( parse_args(argc, argv) == -1 ) {
+      fprintf(stderr, "%s", usage);
+      exit(1);
+    }
+
+    lua = ug_lua_init(ctx.lua_file);
+    if ( !lua )
+      exit(1);
+
+    if ( ctx.in_file ) {
+      file = fopen(ctx.in_file, "r");
+      if ( !file ) {
+        perror(ctx.in_file);
+        exit(1);
+      }
+    } else {
+      file = stdin;
+    }
+
+    while (1) {
+        line_size = getline(&line, &allocated, file);
+        if ( line_size < 0 )
+          break;
+
+        ug_process_line(lua, line, line_size, offset);
+        offset += line_size;
+    }
+    ug_lua_on_eof(lua);
+}