statsrb 0.1.4 → 0.2.0

data/ext/statsrb/statsrb.c

@@ -3,54 +3,358 @@
 #include <string.h>
 #include <stdlib.h>
 
+/**
+ * Keeps track of a single event.
+ */
+typedef struct {
+  int ns_index;
+  int timestamp;
+  int value;
+} StatsrbEvent;
+
+/**
+ * Keeps track of available namespaces.
+ */
+typedef struct {
+  char *namespace[256];
+} StatsrbNS;
+
+/**
+ * Keeps track of internal storage.
+ */
+typedef struct {
+  StatsrbEvent *event_list;
+  int event_count;
+  int event_memory;
+  StatsrbNS *ns_list;
+  int ns_count;
+  int ns_memory;
+} StatsrbInternal;
+
+/**
+ * Internal: retreives the internal storage.
+ */
+static StatsrbInternal* statsrb_get_internal(VALUE self) {
+  StatsrbInternal *internal;
+  Data_Get_Struct(self, StatsrbInternal, internal);
+
+  return internal;
+}
+
+/**
+ * Internal: allocates internal storage.
+ */
+static VALUE statsrb_alloc_internal(VALUE self) {
+  // Allocate internal memory for the StatsrbEvent structs.
+  StatsrbEvent *eventlist = (StatsrbEvent *)calloc(1, sizeof(StatsrbEvent));
+
+  // Allocate memory for the list of namespaces.
+  StatsrbNS *nslist = (StatsrbNS *)calloc(1, sizeof(StatsrbNS));
+
+  // Allocate memory for the pointer storage;
+  StatsrbInternal *internalptr = (StatsrbInternal *)calloc(1, sizeof(StatsrbInternal));
+  internalptr->event_list = eventlist;
+  internalptr->event_count = 0;
+  internalptr->event_memory = 0;
+  internalptr->ns_list = nslist;
+  internalptr->ns_count = 0;
+  internalptr->ns_memory = 0;
+  return Data_Wrap_Struct(self, 0, free, internalptr);
+}
+
+/**
+ * Clears out the internal memory.
+ * @return [void]
+ */
+static void statsrb_data_clear_events(VALUE self) {
+  StatsrbInternal *internal = statsrb_get_internal(self);
+
+  // Allocate internal memory for the StatsrbEvent structs.
+  StatsrbEvent *event_success = (StatsrbEvent *)realloc(internal->event_list, sizeof(StatsrbEvent));
+
+  // Allocate memory for the list of namespaces.
+  StatsrbNS *ns_success = (StatsrbNS *)realloc(internal->ns_list, sizeof(StatsrbNS));
+
+  // Allocate memory for the pointer storage;
+  if (event_success && ns_success) {
+    StatsrbEvent *event_list = event_success;
+    event_success = NULL;
+    StatsrbNS *ns_list = ns_success;
+    ns_success = NULL;
+    internal->event_list = event_list;
+    internal->event_count = 0;
+    internal->event_memory = 0;
+    internal->ns_list = ns_list;
+    internal->ns_count = 0;
+    internal->ns_memory = 0;
+  }
+  else {
+    fprintf(stderr, "Error deallocating memory");
+    return;
+  }
+}
+
+
+/**
+ * Returns the length of the internal storage.
+ * @return [Numeric] The count of items in the internal storage.
+ */
+static VALUE statsrb_length(VALUE self) {
+  StatsrbInternal *internal = statsrb_get_internal(self);
+  if (!internal->event_count) {
+    internal->event_count = 0;
+  }
+
+  return INT2NUM(internal->event_count);
+}
+
+/**
+ * Debugging function.
+ */
+static void statsrb_debug_print_internal(VALUE self) {
+  StatsrbInternal *internal = statsrb_get_internal(self);
+  int i;
+
+  //for (i = 0; i < internal->event_count; i++) {
+    //fprintf(stdout, "Debug: ns: %s; ts: %d; v: %d\n", internal->ns_list[internal->event_list[i].ns_index].namespace, internal->event_list[i].timestamp, internal->event_list[i].value);
+  //}
+  fprintf(stdout, "Debug: count: %d memory: %d\n", internal->event_count, internal->event_memory);
+}
+
+
+/**
+ * Implementation of quicksort algorithm.
+ */
+void time_sort(int left, int right, StatsrbEvent * event_list) {
+  int i = left;
+  int j = right;
+  int p = (i + j) / 2;
+  int pv = event_list[p].timestamp;
+  StatsrbEvent * tmp = (StatsrbEvent *)malloc(sizeof(StatsrbEvent));
+
+  while (i <= j) {
+    while (event_list[i].timestamp < pv) {
+      i++;
+    }
+    while (event_list[j].timestamp > pv) {
+      j--;
+    }
+    if (i <= j) {
+      memcpy(tmp, &event_list[i], sizeof(StatsrbEvent));
+      memcpy(&event_list[i], &event_list[j], sizeof(StatsrbEvent));
+      memcpy(&event_list[j], tmp, sizeof(StatsrbEvent));
+      i++;
+      j--;
+    }
+  }
+
+  free(tmp);
+
+  if (left < j) {
+    time_sort(left, j, event_list);
+  }
+  if (i < right) {
+    time_sort(i, right, event_list);
+  }
+}
+
+/**
+ * Sorts internal data using a quicksort algorithm based on the hash element's timestamp.
+ * @return [Statsrb] A reference to the object.
+ */
+static VALUE statsrb_sort(VALUE self) {
+  StatsrbInternal *internal = statsrb_get_internal(self);
+  if (internal->event_count > 0) {
+    time_sort(0, internal->event_count - 1, internal->event_list);
+  }
+
+  return self;
+}
+
+/**
+ * Internal: pushes a namespace onto the internal storage or retrieves a
+ * preexisting one.
+ *
+ * @param VALUE self
+ * @param const char *namespace
+ *
+ * @return Integer
+ *   The pointer index of the namespace in @nslist.
+ */
+static int statsrb_data_push_ns(VALUE self, const char *namespace) {
+  int i;
+  StatsrbInternal *internal = statsrb_get_internal(self);
+
+  for (i = 0; i < internal->ns_count; i++) {
+    if (strcmp(internal->ns_list[i].namespace, namespace) == 0) {
+      return i;
+    }
+  }
+
+  int memory = (internal->ns_count + 1) * sizeof(StatsrbNS);
+  StatsrbNS *success = (StatsrbNS *)realloc(internal->ns_list, memory);
+
+  if (success) {
+    internal->ns_list = success;
+    success = NULL;
+    strcpy(internal->ns_list[internal->ns_count].namespace, namespace);
+    internal->ns_count++;
+    return internal->ns_count - 1;
+  }
+  else {
+    fprintf(stderr, "Error allocating memory");
+  }
+
+}
+
+/**
+ * Internal: pushes a data event onto the internal storage.
+ *
+ * @param VALUE self
+ * @param const char *namespace
+ * @param int timestamp
+ * @param int value
+ */
+static void statsrb_data_push_event(VALUE self, const char *namespace, int timestamp, int value) {
+  StatsrbInternal *internal = statsrb_get_internal(self);
+
+  // Get the index of the namespace pointer.
+  int ns_index = statsrb_data_push_ns(self, namespace);
+
+  // If it appears that we are approaching the end of the memory block, allocate
+  // some more.
+  // @TODO 2x memory is a little nuts, maybe throttle this back a bit?
+  if ((sizeof(StatsrbEvent) * internal->event_count) > (internal->event_memory * .9)) {
+    internal->event_memory = (2* internal->event_count) * sizeof(StatsrbEvent);
+    StatsrbEvent *success = (StatsrbEvent *)realloc(internal->event_list, internal->event_memory);
+    if (success) {
+      internal->event_list = success;
+      success = NULL;
+    }
+    else {
+      fprintf(stderr, "Error allocating memory");
+      return;
+    }
+  }
+
+  // Set the values;
+  internal->event_list[internal->event_count].timestamp = timestamp;
+  internal->event_list[internal->event_count].ns_index = ns_index;
+  internal->event_list[internal->event_count].value = value;
+
+  // Track the count by saving the new pointer.
+  internal->event_count++;
+}
+
+/**
+ * Creates a ruby hash from event VALUEs.
+ *
+ * @param VALUE self
+ * @param VALUE ts
+ * @param VALUE ns
+ * @param VALUE v
+ */
+VALUE statsrb_create_rb_event_hash(VALUE self, VALUE ts, VALUE ns, VALUE v) {
+  VALUE statsrb_key_ts = rb_iv_get(self, "@key_ts");
+  VALUE statsrb_key_ns = rb_iv_get(self, "@key_ns");
+  VALUE statsrb_key_v = rb_iv_get(self, "@key_v");
+
+  VALUE statsrb_event = rb_hash_new();
+  rb_hash_aset(statsrb_event, statsrb_key_ts, ts);
+  rb_hash_aset(statsrb_event, statsrb_key_ns, ns);
+  rb_hash_aset(statsrb_event, statsrb_key_v, v);
+
+  return statsrb_event;
+}
+
+/**
+ * Pushes a stat onto the statsrb object.
+ * @param timestamp [Number]
+ * @param namespace [String]
+ * @param value [Number]
+ * @return [Statsrb] A reference to the object.
+ */
+static VALUE statsrb_push(VALUE self, VALUE timestamp, VALUE namespace, VALUE value) {
+  int ts = NUM2INT(timestamp);
+  int v = NUM2INT(value);
+  const char *ns = RSTRING_PTR(namespace);
+  statsrb_data_push_event(self, ns, ts, v);
+  return self;
+}
+
 /**
  * Retrieves internal data based on specified filters.
  * @param namespace [String]
  * @param limit [Number]
  * @param start_time [Number]
  * @param end_time [Number]
- * @return [Array] An array of data hashes.
+ * @return [Array] An array of data event hashes.
  */
 static VALUE statsrb_get(VALUE self, VALUE query_ns, VALUE query_limit, VALUE query_start, VALUE query_end) {
-  VALUE statsrb_data = rb_iv_get(self, "@data");
-  VALUE statsrb_event = rb_hash_new();
-  int data_length = RARRAY_LEN(statsrb_data);
-  int i = 0;
-  int count = 0;
-  int tmp_ts;
+  // @TODO maybe it would be sane to make a new statsrb object and then just have
+  // methods to dump everything to ary, json, etc.
+  StatsrbInternal *internal = statsrb_get_internal(self);
+  int tmp_ts, tmp_v, tmp_i;
 
   VALUE filtered_data = rb_ary_new();
-  VALUE tmp_ns;
+  VALUE rb_ns_list = rb_ary_new();
+  VALUE statsrb_event;
 
-  // @data hash key symbols.
-  VALUE statsrb_key_ts = rb_iv_get(self, "@key_ts");
-  VALUE statsrb_key_ns = rb_iv_get(self, "@key_ns");
-  VALUE statsrb_key_v = rb_iv_get(self, "@key_v");
+  int i = 0;
+  int filtered_count = 0;
 
-  // Convert into an int that ruby understands.
   int limit = NUM2INT(query_limit);
   int qstart = NUM2INT(query_start);
   int qend = NUM2INT(query_end);
 
-  for (i = 0; i < data_length; i++) {
-    tmp_ts = NUM2INT(rb_hash_aref(rb_ary_entry(statsrb_data, i), statsrb_key_ts));
-    tmp_ns = rb_hash_aref(rb_ary_entry(statsrb_data, i), statsrb_key_ns);
-    if (rb_str_equal(query_ns, tmp_ns)
-        && (qstart == 0 || tmp_ts >= qstart)
-        && (qend == 0 || tmp_ts <= qend)) {
-      rb_hash_aset(statsrb_event, statsrb_key_ts, rb_hash_aref(rb_ary_entry(statsrb_data, i), statsrb_key_ts));
-      rb_hash_aset(statsrb_event, statsrb_key_ns, rb_hash_aref(rb_ary_entry(statsrb_data, i), statsrb_key_ns));
-      rb_hash_aset(statsrb_event, statsrb_key_v, rb_hash_aref(rb_ary_entry(statsrb_data, i), statsrb_key_v));
+  VALUE rb_ns;
+
+  // Create rb strings for the namespaces.
+  signed int found = -1;
+  for (i = 0; i < internal->ns_count; i++) {
+    rb_hash_aset(rb_ns_list, INT2NUM(i), rb_str_new2(internal->ns_list[i].namespace));
+    if (strcmp(RSTRING_PTR(query_ns), RSTRING_PTR(rb_hash_aref(rb_ns_list, INT2NUM(i)))) == 0) {
+      memcpy(&found, &i, sizeof(int));
+    }
+  }
+
+  // Return right away if the namespace doesn't exist.
+  if (found == -1) {
+    rb_ary_resize(filtered_data, (long) 0);
+    return filtered_data;
+  }
+
+  // Iterate through the in-memory data to find matches.
+  for (i = 0; i < internal->event_count; i++) {
+    if (found == internal->event_list[i].ns_index
+        && (qstart == 0 || internal->event_list[i].timestamp >= qstart)
+        && (qend == 0 || internal->event_list[i].timestamp <= qend)) {
+
+      memcpy(&tmp_ts, &internal->event_list[i].timestamp, sizeof(int));
+      memcpy(&tmp_v, &internal->event_list[i].value, sizeof(int));
+
+      statsrb_event = statsrb_create_rb_event_hash(
+        self,
+        INT2NUM(tmp_ts),
+        rb_hash_aref(rb_ns_list, INT2NUM(found)),
+        INT2NUM(tmp_v)
+      );
+
       rb_ary_push(filtered_data, statsrb_event);
-      count++;
+      filtered_count++;
+    }
+
+    if (limit > 0 && filtered_count == limit) {
+      break;
     }
   }
 
+  rb_ary_resize(filtered_data, filtered_count);
   return filtered_data;
 }
 
 /**
- * Locates data from a specified file and loads into @data.
+ * Locates data from a specified file and loads into internal memory.
  * @param filepath [String]
  * @param namespace [String]
  * @param limit [Number]
@@ -60,28 +364,18 @@ static VALUE statsrb_get(VALUE self, VALUE query_ns, VALUE query_limit, VALUE qu
  */
 static VALUE statsrb_read(VALUE self, VALUE logfile, VALUE query_ns, VALUE query_limit, VALUE query_start, VALUE query_end) {
   FILE * file;
-  int line_size = 256;
+  int line_size = 512;
   char *line = (char *) malloc(line_size);
+  char *tmp_ns = (char *) malloc(256);
   const char *filepath = RSTRING_PTR(logfile);
   const char *query_ns_char = RSTRING_PTR(query_ns);
-
-  // @data hash key symbols.
-  VALUE statsrb_key_ts = rb_iv_get(self, "@key_ts");
-  VALUE statsrb_key_ns = rb_iv_get(self, "@key_ns");
-  VALUE statsrb_key_v = rb_iv_get(self, "@key_v");
-  // Create an empty string for comparison.
-  VALUE statsrb_str_empty = rb_str_new2("");
+  int tmp_v, tmp_ts;
 
   // Convert into an int that ruby understands.
   int limit = NUM2INT(query_limit);
   int qstart = NUM2INT(query_start);
   int qend = NUM2INT(query_end);
 
-  // Return array instantiation.
-  VALUE statsrb_data = rb_iv_get(self, "@data");
-  // @TODO does this garbage collect all of the old hash data?
-  rb_ary_resize(statsrb_data, 0);
-
   file = fopen(filepath, "r");
   if (file == NULL) {
     fprintf(stderr, "File error: could not open file %s for reading.", filepath);
@@ -98,29 +392,28 @@ static VALUE statsrb_read(VALUE self, VALUE logfile, VALUE query_ns, VALUE query
 
     // If the namespace is in the row, explode it.
     if (line[0] != '\0' && line[0] != '\n' && strchr(line, query_ns_char[0]) && strstr(line, query_ns_char)) {
-      VALUE statsrb_event = rb_hash_new();
+      //VALUE statsrb_event = rb_hash_new();
 
       // I tried sscanf for convenience, but it was predictably slower.
       //int statsrb_ts, statsrb_v;
       //sscanf(line, "%d\t%*s\t%d", &statsrb_ts, &statsrb_v);
 
       // @TODO this should something more robust than atoi.
-      int statsrb_ts = atoi(strtok(line, "\t"));
+      tmp_ts = atoi(strtok(line, "\t"));
 
-      if (statsrb_ts != NULL && (qstart == 0 || statsrb_ts >= qstart) && (qend == 0 || statsrb_ts <= qend)) {
+      if (tmp_ts != NULL && (qstart == 0 || tmp_ts >= qstart) && (qend == 0 || tmp_ts <= qend)) {
         // @TODO this should probably use the actual namespace if we do wildcard queries.
-        VALUE statsrb_str_ns = rb_str_new2(strtok(NULL, "\t"));
+        strcpy(tmp_ns, strtok(NULL, "\t"));
         //strtok(NULL, "\t");
-        int statsrb_v = atoi(strtok(NULL, "\0"));
+        tmp_v = atoi(strtok(NULL, "\0"));
 
         // @TODO this should really query the namespace exactly instead of just relying on strstr.
         //if (rb_str_cmp(query_ns, statsrb_str_empty) == 0 || rb_str_cmp(query_ns, statsrb_str_ns) == 0) {
-        if (statsrb_ts && (statsrb_v || statsrb_v == 0)) {
-          rb_hash_aset(statsrb_event, statsrb_key_ts, INT2NUM(statsrb_ts));
-          rb_hash_aset(statsrb_event, statsrb_key_ns, statsrb_str_ns);
-          //rb_hash_aset(statsrb_event, statsrb_key_ns, query_ns);
-          rb_hash_aset(statsrb_event, statsrb_key_v, INT2NUM(statsrb_v));
-          rb_ary_push(statsrb_data, statsrb_event);
+        if (tmp_ts && (tmp_v || tmp_v == 0)) {
+          statsrb_data_push_event(self,
+            tmp_ns,
+            tmp_ts,
+            tmp_v);
           count++;
         }
       }
@@ -130,63 +423,14 @@ static VALUE statsrb_read(VALUE self, VALUE logfile, VALUE query_ns, VALUE query
   // terminate
   fclose (file);
   free (line);
-
-  //return statsrb_data;
-  //rb_iv_set(self, "@data", statsrb_data);
+  free (tmp_ns);
 
   return self;
 }
 
-/**
- * Implementation of quicksort algorithm.
- */
-void time_sort(int left, int right, VALUE ary, VALUE statsrb_key_ts) {
-  int i = left;
-  int j = right;
-  int p = (i + j) / 2;
-  int pv = NUM2INT(rb_hash_aref(rb_ary_entry(ary, p), statsrb_key_ts));
-  VALUE tmp;
-
-  while (i <= j) {
-    while (NUM2INT(rb_hash_aref(rb_ary_entry(ary, i), statsrb_key_ts)) < pv) {
-      i++;
-    }
-    while (NUM2INT(rb_hash_aref(rb_ary_entry(ary, j), statsrb_key_ts)) > pv) {
-      j--;
-    }
-    if (i <= j) {
-      tmp = rb_ary_entry(ary, i);
-      rb_ary_store(ary, i, rb_ary_entry(ary, j));
-      rb_ary_store(ary, j, tmp);
-      i++;
-      j--;
-    }
-  }
-
-  if (left < j) {
-    time_sort(left, j, ary, statsrb_key_ts);
-  }
-  if (i < right) {
-    time_sort(i, right, ary, statsrb_key_ts);
-  }
-}
-
-/**
- * Sorts @data using a quicksort algorithm based on the hash element's timestamp.
- * @return [Hash] The sorted data
- */
-static VALUE statsrb_sort(VALUE self) {
-  VALUE statsrb_data = rb_iv_get(self, "@data");
-  int len = RARRAY_LEN(statsrb_data);
-  if (len > 0) {
-    VALUE statsrb_key_ts = rb_iv_get(self, "@key_ts");
-    time_sort(0, len - 1, statsrb_data, statsrb_key_ts);
-  }
-  return statsrb_data;
-}
 
 /**
- * Writes the @data in memory to a specified file.
+ * Writes the in memory data to a specified file.
  * @param filepath [String]
  * @param filemode [String]
  * @return [Statsrb] A reference to the object.
@@ -195,17 +439,9 @@ static VALUE statsrb_write(VALUE self, VALUE logfile, VALUE mode) {
   FILE * file;
   const char *filepath = RSTRING_PTR(logfile);
   const char *filemode = RSTRING_PTR(mode);
-  VALUE statsrb_data = rb_iv_get(self, "@data");
-  int data_length = RARRAY_LEN(statsrb_data);
-  int i;
-  int line_size = 256;
-  int tmp_ts, tmp_v;
-  const char *tmp_ns = (char *) malloc(line_size);
 
-  // @data hash key symbols.
-  VALUE statsrb_key_ts = rb_iv_get(self, "@key_ts");
-  VALUE statsrb_key_ns = rb_iv_get(self, "@key_ns");
-  VALUE statsrb_key_v = rb_iv_get(self, "@key_v");
+  StatsrbInternal *internal = statsrb_get_internal(self);
+  int i;
 
   file = fopen(filepath, filemode);
   if (file==NULL) {
@@ -213,15 +449,15 @@ static VALUE statsrb_write(VALUE self, VALUE logfile, VALUE mode) {
     return self;
   }
 
-  // Iterate through the data array, writing the data as we go.
-  for (i = 0; i < data_length; i++) {
+  // Iterate through the internal data, writing as we go.
+  for (i = 0; i < internal->event_count; i++) {
     // @TODO make sure that these values are not empty before writing.
-    //VALUE tmp_line = rb_str_tmp_new(line_size);
-    tmp_ts = NUM2INT(rb_hash_aref(rb_ary_entry(statsrb_data, i), statsrb_key_ts));
-    tmp_ns = RSTRING_PTR(rb_hash_aref(rb_ary_entry(statsrb_data, i), statsrb_key_ns));
-    tmp_v = NUM2INT(rb_hash_aref(rb_ary_entry(statsrb_data, i), statsrb_key_v));
-    fprintf(file, "%d\t%s\t%d\n", tmp_ts, tmp_ns, tmp_v);
-    //rb_str_free(tmp_line);
+    fprintf(file,
+            "%d\t%s\t%d\n",
+            internal->event_list[i].timestamp,
+            internal->ns_list[internal->event_list[i].ns_index].namespace,
+            internal->event_list[i].value
+    );
   }
 
   fclose (file);
@@ -229,44 +465,28 @@ static VALUE statsrb_write(VALUE self, VALUE logfile, VALUE mode) {
 }
 
 /**
- * Locates data from a specified file and loads into @data.
+ * Writes the in memory data to a separate files based on namespace.
  * @param filepath [String]
- * @param namespace [String]
- * @param limit [Number]
- * @param start_time [Number]
- * @param end_time [Number]
+ * @param filemode [String]
  * @return [Statsrb] A reference to the object.
 */
 static VALUE statsrb_split_write(VALUE self, VALUE logdir, VALUE mode) {
-  VALUE statsrb_data = rb_iv_get(self, "@data");
-  int len = RARRAY_LEN(statsrb_data);
+  StatsrbInternal *internal = statsrb_get_internal(self);
   int i, ii, ns_len;
 
-  // @data hash key symbols.
-  VALUE statsrb_key_ts = rb_iv_get(self, "@key_ts");
-  VALUE statsrb_key_ns = rb_iv_get(self, "@key_ns");
-  VALUE statsrb_key_v = rb_iv_get(self, "@key_v");
-
-  VALUE ns_list = rb_ary_new();
-
-  for (i = 0; i < len; i++) {
-    if (!rb_ary_includes(ns_list, rb_hash_aref(rb_ary_entry(statsrb_data, i), statsrb_key_ns))) {
-      rb_ary_push(ns_list, rb_hash_aref(rb_ary_entry(statsrb_data, i), statsrb_key_ns));
-    }
-  }
-
-  ns_len = RARRAY_LEN(ns_list);
-
-  for (i = 0; i < ns_len; i++) {
-    VALUE tmp = rb_obj_dup(self);
-    VALUE tmp_data = rb_ary_new();
-    for (ii = 0; ii < len; ii++) {
-      if (rb_str_cmp(rb_ary_entry(ns_list, i), rb_hash_aref(rb_ary_entry(statsrb_data, ii), statsrb_key_ns)) == 0) {
-        rb_ary_push(tmp_data, rb_ary_entry(statsrb_data, ii));
+  VALUE filename;
+  VALUE klass = rb_obj_class(self);
+  VALUE tmp = rb_class_new_instance(0, NULL, klass);
+
+  for (i = 0; i < internal->ns_count; i++) {
+    for (ii = 0; ii < internal->event_count; ii++) {
+      if (strcmp(internal->ns_list[i].namespace, internal->ns_list[internal->event_list[ii].ns_index].namespace) == 0) {
+        statsrb_data_push_event(tmp,
+          internal->ns_list[internal->event_list[ii].ns_index].namespace,
+          internal->event_list[ii].timestamp,
+          internal->event_list[ii].value);
       }
     }
-    //fputs (RSTRING_PTR(rb_obj_as_string(INT2NUM(RARRAY_LEN(tmp_data)))),stderr);
-    rb_iv_set(tmp, "@data", tmp_data);
 
     // If there is no trailing slash on the log dir, add one.
     const char *filepath = RSTRING_PTR(logdir);
@@ -274,7 +494,9 @@ static VALUE statsrb_split_write(VALUE self, VALUE logdir, VALUE mode) {
     if (filepath[len - 1] != '/') {
       logdir = rb_str_plus(logdir, rb_str_new2("/"));
     }
-    statsrb_write(tmp, rb_str_plus(logdir, rb_ary_entry(ns_list, i)), mode);
+    filename = rb_str_new2(internal->ns_list[i].namespace);
+    statsrb_write(tmp, rb_str_plus(logdir, filename), mode);
+    statsrb_data_clear_events(tmp);
   }
 
   return self;
@@ -322,13 +544,6 @@ static VALUE statsrb_rack_call(VALUE self, VALUE env) {
   VALUE response = rb_ary_new();
   VALUE headers = rb_hash_new();
   VALUE body = rb_ary_new();
-  VALUE statsrb_data = rb_iv_get(self, "@data");
-  VALUE statsrb_hash = rb_hash_new();
-
-  // @data hash key symbols.
-  VALUE statsrb_key_ts = rb_iv_get(self, "@key_ts");
-  VALUE statsrb_key_ns = rb_iv_get(self, "@key_ns");
-  VALUE statsrb_key_v = rb_iv_get(self, "@key_v");
 
   char *path = RSTRING_PTR(rb_hash_aref(env, rb_str_new2("PATH_INFO")));
 
@@ -344,7 +559,7 @@ static VALUE statsrb_rack_call(VALUE self, VALUE env) {
   const char *method_getu = "GET";
   const char *method_put = "put";
   const char *method_putu = "PUT";
-  // Remove the leading /
+  // Remove the leading slash.
   path++;
   const char *method = strtok(path, "/\0");
   if (method && (strcmp(method, method_put) == 0 || strcmp(method, method_putu) == 0)) {
@@ -373,18 +588,16 @@ static VALUE statsrb_rack_call(VALUE self, VALUE env) {
         statsrb_v = atoi(RSTRING_PTR(statsrb_v_qs));
       }
 
-      rb_hash_aset(statsrb_hash, statsrb_key_ts, INT2NUM(statsrb_ts));
-      rb_hash_aset(statsrb_hash, statsrb_key_ns, statsrb_ns);
-      rb_hash_aset(statsrb_hash, statsrb_key_v, INT2NUM(statsrb_v));
-      rb_ary_push(statsrb_data, statsrb_hash);
+      statsrb_data_push_event(self, RSTRING_PTR(statsrb_ns), statsrb_ts, statsrb_v);
+
+      int data_length = NUM2INT(statsrb_length(self));
 
-      int data_length = RARRAY_LEN(statsrb_data);
-      rb_ary_push(body, rb_obj_as_string(INT2NUM(RARRAY_LEN(statsrb_data))));
+      rb_ary_push(body, rb_obj_as_string(INT2NUM(data_length)));
 
-      if (data_length > NUM2INT(rb_iv_get(self, "@flush_count"))) {
+      if (data_length >= NUM2INT(rb_iv_get(self, "@flush_count"))) {
         statsrb_sort(self);
         statsrb_split_write(self, rb_iv_get(self, "@split_file_dir"), rb_str_new2("a+"));
-        rb_ary_resize(statsrb_data, 0);
+        statsrb_data_clear_events(self);
       }
 
       rb_ary_push(body, statsrb_ns);
@@ -400,6 +613,7 @@ static VALUE statsrb_rack_call(VALUE self, VALUE env) {
     if (jsoncallback != Qnil) {
       rb_ary_push(body, rb_str_plus(jsoncallback, rb_str_new("(", 1)));
     }
+    // @TODO move this to a to_json method.
     char json_start[256];
     sprintf(json_start, "{\"%s\":[", statsrb_str_ns);
     rb_ary_push(body, rb_str_new2(json_start));
@@ -440,27 +654,30 @@ static VALUE statsrb_rack_call(VALUE self, VALUE env) {
       }
 
       // Create a new Statsrb object to query from.
-      // @todo we probably need to assign a new array to @data to avoid messing up the pointers.
-      VALUE tmp = rb_obj_dup(self);
-      VALUE tmp_data = rb_ary_new();
-      rb_iv_set(tmp, "@data", tmp_data);
+      VALUE klass = rb_obj_class(self);
+      VALUE tmp = rb_class_new_instance(0, NULL, klass);
+
       statsrb_read(tmp, rb_str_plus(rb_iv_get(self, "@split_file_dir"), statsrb_ns), statsrb_ns, INT2NUM(query_limit), INT2NUM(query_start), INT2NUM(query_end));
       statsrb_sort(tmp);
 
-      int i, data_length = RARRAY_LEN(tmp_data);
+      int i, data_length = NUM2INT(statsrb_length(tmp));
+      StatsrbInternal *internal = statsrb_get_internal(tmp);
 
       for (i = 0; i < data_length; i++) {
         rb_ary_push(body, rb_str_new("[", 1));
-        rb_ary_push(body, rb_obj_as_string(rb_hash_aref(rb_ary_entry(tmp_data, i), statsrb_key_ts )));
-        rb_ary_push(body, rb_str_new(",", 1));
-        rb_ary_push(body, rb_obj_as_string(rb_hash_aref(rb_ary_entry(tmp_data, i), statsrb_key_v )));
+        rb_ary_push(body, rb_obj_as_string(INT2NUM(internal->event_list[i].timestamp)));
+        rb_ary_push(body, rb_str_new(",\"", 2));
+        rb_ary_push(body, rb_str_new2(internal->ns_list[internal->event_list[i].ns_index].namespace));
+        rb_ary_push(body, rb_str_new("\",", 2));
+        rb_ary_push(body, rb_obj_as_string(INT2NUM(internal->event_list[i].value)));
         rb_ary_push(body, rb_str_new("]", 1));
+
         if (i < data_length - 1) {
           rb_ary_push(body, rb_str_new(",", 1));
         }
         rb_ary_push(body, rb_str_new("\n", 1));
       }
-      rb_ary_resize(tmp_data, 0);
+      statsrb_data_clear_events(tmp);
     }
     rb_ary_push(body, rb_str_new("]}", 2));
     if (jsoncallback != Qnil) {
@@ -482,36 +699,27 @@ static VALUE statsrb_rack_call(VALUE self, VALUE env) {
 }
 
 /**
- * Pushes a stat onto the statsrb object.
- * @param timestamp [Number]
+ * Populates the internal storage with test data.
+ *
  * @param namespace [String]
- * @param value [Number]
- * @return [Statsrb] A reference to the object.
+ * @param amount [Numeric]
  */
-static VALUE statsrb_push(VALUE self, VALUE timestamp, VALUE namespace, VALUE value) {
-  VALUE statsrb_data = rb_iv_get(self, "@data");
-  VALUE statsrb_event = rb_hash_new();
-
-  // @data hash key symbols.
-  VALUE statsrb_key_ts = rb_iv_get(self, "@key_ts");
-  VALUE statsrb_key_ns = rb_iv_get(self, "@key_ns");
-  VALUE statsrb_key_v = rb_iv_get(self, "@key_v");
-
-  rb_hash_aset(statsrb_event, statsrb_key_ts, timestamp);
-  rb_hash_aset(statsrb_event, statsrb_key_ns, namespace);
-  rb_hash_aset(statsrb_event, statsrb_key_v, value);
-
-  rb_ary_push(statsrb_data, statsrb_event);
-
-  return self;
+static void statsrb_load_test(VALUE self, VALUE ns, VALUE amt) {
+  StatsrbInternal *internal = statsrb_get_internal(self);
+  int i, val;
+  srand(time(NULL));
+  for (i = 0; i < NUM2INT(amt); i++) {
+    val = rand();
+    statsrb_data_push_event(self, RSTRING_PTR(ns), val + 100, val + 1);
+  }
+  statsrb_debug_print_internal(self);
+  fprintf(stdout, "Debug: count: %d\n", internal->event_count);
 }
 
 /**
  * Class constructor, sets up an instance variable.
  */
 static VALUE statsrb_constructor(VALUE self) {
-  VALUE statsrb_data = rb_ary_new();
-  rb_iv_set(self, "@data", statsrb_data);
   VALUE statsrb_split_file_dir = rb_str_new("/tmp", 4);
   rb_iv_set(self, "@split_file_dir", statsrb_split_file_dir);
   rb_iv_set(self, "@flush_count", INT2NUM(9));
@@ -535,21 +743,21 @@ void Init_statsrb(void) {
   VALUE klass = rb_define_class("Statsrb", rb_cObject);
 
   // Instance methods and properties.
+  rb_define_alloc_func(klass, statsrb_alloc_internal);
   rb_define_method(klass, "initialize", statsrb_constructor, 0);
   rb_define_method(klass, "query", statsrb_read, 5);
   rb_define_method(klass, "read", statsrb_read, 5);
   rb_define_method(klass, "get", statsrb_get, 4);
+  rb_define_method(klass, "load_test", statsrb_load_test, 2);
+  rb_define_method(klass, "length", statsrb_length, 0);
   rb_define_method(klass, "sort", statsrb_sort, 0);
   rb_define_method(klass, "write", statsrb_write, 2);
   rb_define_method(klass, "split_write", statsrb_split_write, 2);
   rb_define_method(klass, "push", statsrb_push, 3);
+  rb_define_method(klass, "clear", statsrb_data_clear_events, 0);
   rb_define_method(klass, "call", statsrb_rack_call, 1);
-  // Define :attr_accessor (read/write instance var)
-  // Note that this must correspond with a call to rb_iv_self() and it's string name must be @data.
-  // An array of hashes keyed with :ts(timestamp), :ns(namespace) and :v(value) e.g. [!{:ts => Time.now.to_i, :ns => "test", :v => 33}]
-  rb_define_attr(klass, "data", 1, 1);
   // The file directory to write when splitting namespaces. @see #split_write
   rb_define_attr(klass, "split_file_dir", 1, 1);
-  // When used with a rack server, the max count of @data before flushing and writing to file.
+  // When used with a rack server, the max count of internal events.
   rb_define_attr(klass, "flush_count", 1, 1);
 }

data/test/test_statsrb.rb

@@ -1,5 +1,7 @@
 require 'minitest/autorun'
 require 'statsrb'
+require 'json'
+require 'pp'
 
 class TestStatsrb < MiniTest::Test
 
@@ -8,10 +10,14 @@ class TestStatsrb < MiniTest::Test
   def setup
     @s = Statsrb.new
     @tmpfile = "/tmp/test.statsrb"
+    @s.split_file_dir = "/tmp/"
+    @s.flush_count = 10
   end
 
   def teardown
     File.delete @tmpfile unless !File.exists? @tmpfile
+    rackfile = "/tmp/test"
+    File.delete rackfile unless !File.exists? rackfile
   end
 
   # Provides test data.
@@ -38,12 +44,14 @@ class TestStatsrb < MiniTest::Test
   # Tests that the data was indeed pushed.
   def test_push_data
     push_data
-    assert_equal @s.data.length, get_data.length
+    assert_equal @s.length, get_data.length
   end
-  
+
   # Tests that we can filter the in-memory data.
   def test_get_data
     push_data
+    t = @s.get "test1", 100, 0, 0
+    assert_equal(t.length, 3);
     t = @s.get "test2", 100, 0, 0
     assert_equal(t.length, 2);
   end
@@ -53,7 +61,8 @@ class TestStatsrb < MiniTest::Test
     current = 0
     push_data
     @s.sort
-    @s.data.each do |value|
+    t = @s.get "test1", 100, 0, 0
+    t.each do |value|
       assert value[:ts] > current
       current = value[:ts]
     end
@@ -77,13 +86,86 @@ class TestStatsrb < MiniTest::Test
     end
   end
 
+  # Tests that we can clear data from memory.
+  def test_clear_data
+    push_data
+    assert_equal @s.length, get_data.length
+    @s.clear
+    assert_equal @s.length, 0
+  end
+
   # Tests that we can read data from a file.
   def test_read_data
     push_data
     write_data
+    @s.clear
     @s.read @tmpfile, "test1", 100, 0, 0
-    assert_equal @s.data.length, 3
+    assert_equal @s.length, 3
+    @s.clear
     @s.read @tmpfile, "test2", 100, 0, 0
-    assert_equal @s.data.length, 2
+    assert_equal @s.length, 2
+  end
+
+  # Tests that the rack interface works properly.
+  def test_rack_call
+    # Test putting data.
+    env = {
+      "PATH_INFO" => "/PUT",
+      "QUERY_STRING" => "name=test&value=13"
+    }
+
+    5.times do |i|
+      @s.call(env);
+    end
+
+    assert_equal 5, @s.length
+
+    # Write enough data to flush.
+    5.times do |i|
+      @s.call(env);
+    end
+
+    # Test getting data.
+    env = {
+      "PATH_INFO" => "/GET/test",
+      "QUERY_STRING" => ""
+    }
+
+    resp = @s.call(env)
+    data = JSON.parse(resp[2].join)
+    assert_equal data["test"].length, 10
+  end
+
+  def test_no_results
+    push_data
+    t = @s.get "noresults", 100, 0, 0
+    assert_equal(t.length, 0);
+  end
+
+  # Tests large data volumes.
+  def test_large_data
+    # Load a lot of data.
+    @s.load_test "kevin", 500000
+    @s.load_test "melissa", 500000
+    @s.load_test "benjamin", 500000
+    @s.sort
+    # Extract all of one namespace.
+    t = @s.get "melissa", 100000, 0, 0
+    assert_equal t.length, 100000
+    # Push them back to the object.
+    t.each do |i|
+      @s.push i[:ts], i[:ns], i[:v]
+    end
+    # Save it to file and clear it.
+    @s.write @tmpfile, "w+"
+    @s.clear
+    # Re-load the data.
+    @s.read @tmpfile, "melissa", 600000, 0, 0
+    # Try to get one that doesn't exist.
+    t = @s.get "kevin", 10000, 0, 0
+    assert_equal t.length, 0
+    # Try to get all of the data out.
+    t = @s.get "melissa", 600000, 0, 0
+    assert_equal t.length, 600000
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: statsrb
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,10 +9,10 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-08 00:00:00.000000000 Z
+date: 2013-06-28 00:00:00.000000000 Z
 dependencies: []
-description: A Ruby time series stats repository using flat file storage, providing
-  a Ruby API as well as a Rack compatible REST API.
+description: A Ruby time series stats repository written in C, using flat file storage,
+  providing a Ruby API as well as a Rack compatible REST API.
 email: email@kevinhankens.com
 executables: []
 extensions: