nmatrix 0.0.8 → 0.0.9

data/ext/nmatrix/storage/{yale.h → yale/yale.h}

@@ -43,18 +43,16 @@
  */
 
 #include <limits> // for std::numeric_limits<T>::max()
+#include <stdexcept>
 
 /*
  * Project Includes
  */
 
-#include "types.h"
-
-#include "data/data.h"
-
-#include "common.h"
-
-#include "nmatrix.h"
+#include "../../types.h"
+#include "../../data/data.h"
+#include "../common.h"
+#include "../../nmatrix.h"
 
 extern "C" {
 
@@ -62,8 +60,6 @@ extern "C" {
    * Macros
    */
 
-  #define NM_YALE_MINIMUM(sptr)               (((YALE_STORAGE*)(sptr))->shape[0]*2 + 1) // arbitrarily defined
-
   #ifndef NM_CHECK_ALLOC
    #define NM_CHECK_ALLOC(x) if (!x) rb_raise(rb_eNoMemError, "insufficient memory");
   #endif
@@ -86,13 +82,13 @@ extern "C" {
   // Lifecycle //
   ///////////////
 
-  YALE_STORAGE* nm_yale_storage_create(nm::dtype_t dtype, size_t* shape, size_t dim, size_t init_capacity, nm::itype_t itype);
-  YALE_STORAGE* nm_yale_storage_create_from_old_yale(nm::dtype_t dtype, size_t* shape, void* ia, void* ja, void* a, nm::dtype_t from_dtype);
+  YALE_STORAGE* nm_yale_storage_create(nm::dtype_t dtype, size_t* shape, size_t dim, size_t init_capacity);
+  YALE_STORAGE* nm_yale_storage_create_from_old_yale(nm::dtype_t dtype, size_t* shape, char* ia, char* ja, char* a, nm::dtype_t from_dtype);
   YALE_STORAGE*	nm_yale_storage_create_merged(const YALE_STORAGE* merge_template, const YALE_STORAGE* other);
   void          nm_yale_storage_delete(STORAGE* s);
   void          nm_yale_storage_delete_ref(STORAGE* s);
   void					nm_yale_storage_init(YALE_STORAGE* s, void* default_val);
-  void					nm_yale_storage_mark(void*);
+  void					nm_yale_storage_mark(STORAGE*);
 
   ///////////////
   // Accessors //
@@ -100,9 +96,12 @@ extern "C" {
 
   VALUE nm_yale_each_with_indices(VALUE nmatrix);
   VALUE nm_yale_each_stored_with_indices(VALUE nmatrix);
-  void* nm_yale_storage_get(STORAGE* s, SLICE* slice);
-  void*	nm_yale_storage_ref(STORAGE* s, SLICE* slice);
-  char  nm_yale_storage_set(STORAGE* storage, SLICE* slice, void* v);
+  VALUE nm_yale_stored_diagonal_each_with_indices(VALUE nmatrix);
+  VALUE nm_yale_stored_nondiagonal_each_with_indices(VALUE nmatrix);
+  VALUE nm_yale_each_ordered_stored_with_indices(VALUE nmatrix);
+  void* nm_yale_storage_get(const STORAGE* s, SLICE* slice);
+  void*	nm_yale_storage_ref(const STORAGE* s, SLICE* slice);
+  void  nm_yale_storage_set(VALUE left, SLICE* slice, VALUE right);
 
   //char  nm_yale_storage_vector_insert(YALE_STORAGE* s, size_t pos, size_t* js, void* vals, size_t n, bool struct_only, nm::dtype_t dtype, nm::itype_t itype);
   //void  nm_yale_storage_increment_ia_after(YALE_STORAGE* s, size_t ija_size, size_t i, size_t n);
@@ -128,39 +127,6 @@ extern "C" {
   // Utility //
   /////////////
 
-  /*
-   * Calculates the itype a YALE_STORAGE object would need without actually needing
-   * to see the YALE_STORAGE object. Does this just by looking at the shape.
-   *
-   * Useful for creating Yale Storage by other means than NMatrix.new(:yale, ...),
-   * e.g., from a MATLAB v5 .mat file.
-   */
-  inline nm::itype_t nm_yale_storage_itype_by_shape(const size_t* shape) {
-    uint64_t yale_max_size = shape[0] * (shape[1]+1);
-
-    if (yale_max_size < static_cast<uint64_t>(std::numeric_limits<uint8_t>::max()) - 2) {
-      return nm::UINT8;
-
-    } else if (yale_max_size < static_cast<uint64_t>(std::numeric_limits<uint16_t>::max()) - 2) {
-      return nm::UINT16;
-
-    } else if (yale_max_size < std::numeric_limits<uint32_t>::max() - 2) {
-      return nm::UINT32;
-
-    } else {
-      return nm::UINT64;
-    }
-  }
-
-  /*
-   * Determine the index dtype (which will be used for the ija vector). This is
-   * determined by matrix shape, not IJA/A vector capacity. Note that it's MAX-2
-   * because UINTX_MAX and UINTX_MAX-1 are both reserved for sparse matrix
-   * multiplication.
-   */
-  inline nm::itype_t nm_yale_storage_default_itype(const YALE_STORAGE* s) {
-    return nm_yale_storage_itype_by_shape(s->shape);
-  }
 
 
   /////////////////////////
@@ -179,19 +145,21 @@ extern "C" {
 
 } // end of extern "C" block
 
-namespace nm { namespace yale_storage {
+namespace nm {
+
+namespace yale_storage {
 
   /*
-   * Constants
+   * Typedefs
    */
-  const float GROWTH_CONSTANT = 1.5;
+
+  typedef size_t IType;
 
 
   /*
    * Templated Functions
    */
 
-  template <typename IType>
   int binary_search(YALE_STORAGE* s, IType left, IType right, IType key);
 
   /*
@@ -217,14 +185,14 @@ namespace nm { namespace yale_storage {
     }
   }
 
-  template <typename DType, typename IType>
+  template <typename DType>
   void init(YALE_STORAGE* s, void* init_val);
 
-  template <typename IType>
   size_t  get_size(const YALE_STORAGE* storage);
 
-  template <typename IType>
   IType binary_search_left_boundary(const YALE_STORAGE* s, IType left, IType right, IType bound);
+
+
 }} // end of namespace nm::yale_storage
 
 #endif // YALE_H

data/ext/nmatrix/types.h

@@ -49,4 +49,6 @@
 typedef float		float32_t;
 typedef double	float64_t;
 
+typedef size_t  IType;
+
 #endif

data/ext/nmatrix/util/io.cpp

@@ -74,8 +74,8 @@ namespace nm { namespace io {
 template <typename DType, typename MDType>
 char* matlab_cstring_to_dtype_string(size_t& result_len, const char* str, size_t bytes) {
 
-  result_len   = bytes / sizeof(DType);
-  char* result = ALLOC_N(char, bytes / sizeof(DType));
+  result_len   = sizeof(DType) * bytes / sizeof(MDType);
+  char* result = ALLOC_N(char, result_len);
 
   if (bytes % sizeof(MDType) != 0) {
     rb_raise(rb_eArgError, "the given string does not divide evenly for the given MATLAB dtype");
@@ -109,7 +109,7 @@ nm::dtype_t nm_dtype_from_rbstring(VALUE str) {
   	}
   }
 
-  rb_raise(rb_eArgError, "Invalid data type string specified.");
+  rb_raise(rb_eArgError, "invalid data type string (%s) specified", RSTRING_PTR(str));
 }
 
 
@@ -117,31 +117,19 @@ nm::dtype_t nm_dtype_from_rbstring(VALUE str) {
  * Converts a symbol to a data type.
  */
 nm::dtype_t nm_dtype_from_rbsymbol(VALUE sym) {
+  ID sym_id = SYM2ID(sym);
 
   for (size_t index = 0; index < NM_NUM_DTYPES; ++index) {
-    if (SYM2ID(sym) == rb_intern(DTYPE_NAMES[index])) {
+    if (sym_id == rb_intern(DTYPE_NAMES[index])) {
     	return static_cast<nm::dtype_t>(index);
     }
   }
 
-  rb_raise(rb_eArgError, "Invalid data type symbol specified.");
+  VALUE str = rb_any_to_s(sym);
+  rb_raise(rb_eArgError, "invalid data type symbol (:%s) specified", RSTRING_PTR(str));
 }
 
 
-/*
- * Converts a symbol to an index type.
- */
-nm::itype_t nm_itype_from_rbsymbol(VALUE sym) {
-
-  for (size_t index = 0; index < NM_NUM_ITYPES; ++index) {
-    if (SYM2ID(sym) == rb_intern(ITYPE_NAMES[index])) {
-    	return static_cast<nm::itype_t>(index);
-    }
-  }
-
-  rb_raise(rb_eArgError, "Invalid index type specified.");
-}
-
 /*
  * Converts a string to a storage type. Only looks at the first three
  * characters.
@@ -169,7 +157,8 @@ nm::stype_t nm_stype_from_rbsymbol(VALUE sym) {
     }
   }
 
-  rb_raise(rb_eArgError, "Invalid storage type symbol specified");
+  VALUE str = rb_any_to_s(sym);
+  rb_raise(rb_eArgError, "invalid storage type symbol (:%s) specified", RSTRING_PTR(str));
   return nm::DENSE_STORE;
 }
 
@@ -197,35 +186,28 @@ static nm::io::matlab_dtype_t matlab_dtype_from_rbsymbol(VALUE sym) {
  * Arguments:
  * * str        :: the data
  * * from       :: symbol representing MATLAB data type (e.g., :miINT8)
- * * options    :: hash, give either :itype => some_itype or :dtype => some_dtype, tells function
- *                 what to give as output.
+ * * type       :: either :itype or some dtype symbol (:byte, :uint32, etc)
  */
-static VALUE nm_rbstring_matlab_repack(VALUE self, VALUE str, VALUE from, VALUE options) {
+static VALUE nm_rbstring_matlab_repack(VALUE self, VALUE str, VALUE from, VALUE type) {
   nm::io::matlab_dtype_t from_type = matlab_dtype_from_rbsymbol(from);
   uint8_t to_type;
 
-  if (TYPE(options) != T_HASH) {
-    rb_raise(rb_eArgError, "third argument to repack must be an options hash");
-  }
-
-  if (RB_HASH_HAS_SYMBOL_KEY(options, "dtype")) { // Hash#has_key?(:dtype)
-
-    nm::dtype_t to_dtype  = nm_dtype_from_rbsymbol(rb_hash_aref(options, ID2SYM(rb_intern("dtype"))));
-    to_type               = static_cast<int8_t>(to_dtype);
-
-  } else if (RB_HASH_HAS_SYMBOL_KEY(options, "itype")) {
-
-    nm::itype_t to_itype  = nm_itype_from_rbsymbol(rb_hash_aref(options, ID2SYM(rb_intern("itype"))));
-
-    // we're going to cheat and use the DTYPE template table. To do this, we just act like uint8_t
-    // is a dtype (both are 0, itype and dtype), or we add 1 to the other itypes and treat them as
-    // signed.
-    to_type           = static_cast<uint8_t>(to_itype);
-    if (to_itype != nm::UINT8) to_type += 1;
-
-
+  if (SYMBOL_P(type)) {
+    if (rb_to_id(type) == rb_intern("itype")) {
+      if (sizeof(size_t) == sizeof(int64_t)) {
+        to_type = static_cast<int8_t>(nm::INT64);
+      } else if (sizeof(size_t) == sizeof(int32_t)) {
+        to_type = static_cast<int8_t>(nm::INT32);
+      } else if (sizeof(size_t) == sizeof(int16_t)) {
+        to_type = static_cast<int8_t>(nm::INT16);
+      } else {
+        rb_raise(rb_eStandardError, "unhandled size_t definition");
+      }
+    } else {
+      to_type = static_cast<uint8_t>(nm_dtype_from_rbsymbol(type));
+    }
   } else {
-    rb_raise(rb_eArgError, "third argument must have either :itype or :dtype as a key");
+    rb_raise(rb_eArgError, "expected symbol for third argument");
   }
 
   // For next few lines, see explanation above NM_MATLAB_DTYPE_TEMPLATE_TABLE definition in io.h.
@@ -241,7 +223,7 @@ static VALUE nm_rbstring_matlab_repack(VALUE self, VALUE str, VALUE from, VALUE
 
   // Encode as 8-bit ASCII with a length -- don't want to hiccup on \0
   VALUE result = rb_str_new(repacked_data, repacked_data_length);
-  free(repacked_data); // Don't forget to free what we allocated!
+  xfree(repacked_data); // Don't forget to free what we allocated!
 
   return result;
 }

data/ext/nmatrix/util/io.h

@@ -41,7 +41,6 @@
  * Extern Types
  */
 extern const char* const DTYPE_NAMES[nm::NUM_DTYPES];
-extern const char* const ITYPE_NAMES[nm::NUM_ITYPES];
 
 namespace nm { namespace io {
   /*
@@ -77,7 +76,6 @@ extern "C" {
   nm::dtype_t nm_dtype_from_rbstring(VALUE str);
   nm::stype_t nm_stype_from_rbsymbol(VALUE sym);
   nm::stype_t nm_stype_from_rbstring(VALUE str);
-  nm::itype_t nm_itype_from_rbsymbol(VALUE sym);
 
   void nm_init_io(void);
 

data/ext/nmatrix/util/sl_list.cpp

@@ -77,18 +77,18 @@ void del(LIST* list, size_t recursions) {
 
     if (recursions == 0) {
       //fprintf(stderr, "    free_val: %p\n", curr->val);
-      free(curr->val);
+      xfree(curr->val);
       
     } else {
       //fprintf(stderr, "    free_list: %p\n", list);
       del((LIST*)curr->val, recursions - 1);
     }
 
-    free(curr);
+    xfree(curr);
     curr = next;
   }
   //fprintf(stderr, "    free_list: %p\n", list);
-  free(list);
+  xfree(list);
 }
 
 /*
@@ -116,6 +116,37 @@ void mark(LIST* list, size_t recursions) {
 // Accessors //
 ///////////////
 
+
+/*
+ * Given a list, insert key/val as the first entry in the list. Does not do any
+ * checks, just inserts.
+ */
+NODE* insert_first_node(LIST* list, size_t key, void* val, size_t val_size) {
+  NODE* ins   = ALLOC(NODE);
+  ins->next   = list->first;
+
+  void* val_copy = ALLOC_N(char, val_size);
+  memcpy(val_copy, val, val_size);
+
+  ins->val    = reinterpret_cast<void*>(val_copy);
+  ins->key    = key;
+  list->first = ins;
+
+  return ins;
+}
+
+NODE* insert_first_list(LIST* list, size_t key, LIST* l) {
+  NODE* ins   = ALLOC(NODE);
+  ins->next   = list->first;
+
+  ins->val    = reinterpret_cast<void*>(l);
+  ins->key    = key;
+  list->first = ins;
+
+  return ins;
+}
+
+
 /* 
  * Given a list and a key/value-ptr pair, create a node (and return that node).
  * If NULL is returned, it means insertion failed.
@@ -156,11 +187,11 @@ NODE* insert(LIST* list, bool replace, size_t key, void* val) {
   if (ins->key == key) {
     // key already exists
     if (replace) {
-      free(ins->val);
+      xfree(ins->val);
       ins->val = val;
       
     } else {
-    	free(val);
+    	xfree(val);
     }
     
     return ins;
@@ -170,14 +201,14 @@ NODE* insert(LIST* list, bool replace, size_t key, void* val) {
   }
 }
 
+
+
 /*
  * Documentation goes here.
  */
 NODE* insert_after(NODE* node, size_t key, void* val) {
-  NODE* ins;
-
   //if (!(ins = malloc(sizeof(NODE)))) return NULL;
-  ins = ALLOC(NODE);
+  NODE* ins = ALLOC(NODE);
 
   // insert 'ins' between 'node' and 'node->next'
   ins->next  = node->next;
@@ -190,22 +221,69 @@ NODE* insert_after(NODE* node, size_t key, void* val) {
   return ins;
 }
 
+
 /*
- * Analog functions list_insert but this insert copy of value.
+ * Insert a new node immediately after +node+, or replace the existing one if its key is a match.
  */
-NODE* insert_with_copy(LIST *list, size_t key, void *val, size_t size) {
+NODE* replace_insert_after(NODE* node, size_t key, void* val, bool copy, size_t copy_size) {
+  if (node->next && node->next->key == key) {
+
+    // Should we copy into the current one or free and insert?
+    if (copy) memcpy(node->next->val, val, copy_size);
+    else {
+      xfree(node->next->val);
+      node->next->val = val;
+    }
+
+    return node->next;
+
+  } else { // no next node, or if there is one, it's greater than the current key
+
+    if (copy) {
+      void* val_copy = ALLOC_N(char, copy_size);
+      memcpy(val_copy, val, copy_size);
+      return insert_after(node, key, val_copy);
+    } else {
+      return insert_after(node, key, val);
+    }
+
+  }
+}
+
+
+
+/*
+ * Functions analogously to list::insert but this inserts a copy of the value instead of the original.
+ */
+NODE* insert_copy(LIST *list, bool replace, size_t key, void *val, size_t size) {
   void *copy_val = ALLOC_N(char, size);
   memcpy(copy_val, val, size);
 
+  return insert(list, replace, key, copy_val);
+}
+
 
-  return insert(list, false, key, copy_val);
+/*
+ * Returns the value pointer for some key. Doesn't free the memory for that value. Doesn't require a find operation,
+ * assumes finding has already been done. If rm is the first item in the list, prev should be NULL.
+ */
+void* remove_by_node(LIST* list, NODE* prev, NODE* rm) {
+  if (!prev)  list->first = rm->next;
+  else        prev->next  = rm->next;
+
+  void* val   = rm->val;
+  xfree(rm);
+
+  return val;
 }
+
+
 /*
  * Returns the value pointer (not the node) for some key. Note that it doesn't
  * free the memory for the value stored in the node -- that pointer gets
  * returned! Only the node is destroyed.
  */
-void* remove(LIST* list, size_t key) {
+void* remove_by_key(LIST* list, size_t key) {
   NODE *f, *rm;
   void* val;
 
@@ -218,12 +296,12 @@ void* remove(LIST* list, size_t key) {
     rm  = list->first;
     
     list->first = rm->next;
-    free(rm);
+    xfree(rm);
     
     return val;
   }
 
-  f = find_preceding_from(list->first, key);
+  f = find_preceding_from_node(list->first, key);
   if (!f || !f->next) { // not found, end of list
   	return NULL;
   }
@@ -235,7 +313,7 @@ void* remove(LIST* list, size_t key) {
 
     // get the value and free the memory for the node
     val = rm->val;
-    free(rm);
+    xfree(rm);
 
     return val;
   }
@@ -244,29 +322,56 @@ void* remove(LIST* list, size_t key) {
 }
 
 
+bool node_is_within_slice(NODE* n, size_t coord, size_t len) {
+  if (!n) return false;
+  if (n->key >= coord && n->key < coord + len) return true;
+  else return false;
+}
+
+
 /*
  * Recursive removal of lists that may contain sub-lists. Stores the value ultimately removed in rm.
- *
- * FIXME: Could be made slightly faster by using a variety of find which also returns the previous node. This way,
- * FIXME: we can remove directly instead of calling remove() and doing the search over again.
  */
-bool remove_recursive(LIST* list, const size_t* coords, const size_t* offset, size_t r, const size_t& dim, void* rm) {
+bool remove_recursive(LIST* list, const size_t* coords, const size_t* offsets, const size_t* lengths, size_t r, const size_t& dim) {
+//  std::cerr << "remove_recursive: " << r << std::endl;
+  // find the current coordinates in the list
+  NODE* prev    = find_preceding_from_list(list, coords[r] + offsets[r]);
+  NODE* n;
+  if (prev) n  = prev->next && node_is_within_slice(prev->next, coords[r] + offsets[r], lengths[r]) ? prev->next : NULL;
+  else      n  = node_is_within_slice(list->first, coords[r] + offsets[r], lengths[r]) ? list->first : NULL;
 
   if (r < dim-1) { // nodes here are lists
-    // find the current coordinates in the list
-    NODE* n = find(list, coords[r] + offset[r]);
 
-    if (n) {
+    while (n) {
       // from that sub-list, call remove_recursive.
-      bool remove_parent = remove_recursive(reinterpret_cast<LIST*>(n->val), coords, offset, r+1, dim, rm);
+      bool remove_parent = remove_recursive(reinterpret_cast<LIST*>(n->val), coords, offsets, lengths, r+1, dim);
 
       if (remove_parent) { // now empty -- so remove the sub-list
-        free(remove(list, coords[r] + offset[r]));
+//        std::cerr << r << ": removing parent list at " << n->key << std::endl;
+        xfree(remove_by_node(list, prev, n));
+
+        if (prev) n  = prev->next && node_is_within_slice(prev->next, coords[r] + offsets[r], lengths[r]) ? prev->next : NULL;
+        else      n  = node_is_within_slice(list->first, coords[r] + offsets[r], lengths[r]) ? list->first : NULL;
+      } else {
+        // Move forward to next node (list at n still exists)
+        prev         = n;
+        n            = prev->next && node_is_within_slice(prev->next, coords[r] + offsets[r], lengths[r]) ? prev->next : NULL;
       }
+
+      // Iterate to next one.
+      if (prev) n  = prev->next && node_is_within_slice(prev->next, coords[r] + offsets[r], lengths[r]) ? prev->next : NULL;
+      else      n  = node_is_within_slice(list->first, coords[r] + offsets[r], lengths[r]) ? list->first : NULL;
     }
 
   } else { // nodes here are not lists, but actual values
-    rm = remove(list, coords[r] + offset[r]);
+
+    while (n) {
+//      std::cerr << r << ": removing node at " << n->key << std::endl;
+      xfree(remove_by_node(list, prev, n));
+
+      if (prev) n  = prev->next && node_is_within_slice(prev->next, coords[r] + offsets[r], lengths[r]) ? prev->next : NULL;
+      else      n  = node_is_within_slice(list->first, coords[r] + offsets[r], lengths[r]) ? list->first : NULL;
+    }
   }
 
   if (!list->first) return true; // if current list is now empty, signal its removal
@@ -303,21 +408,57 @@ NODE* find(LIST* list, size_t key) {
   return NULL;
 }
 
+
+
+/*
+ * Find some element in the list and return the node ptr for that key.
+ */
+NODE* find_with_preceding(LIST* list, size_t key, NODE*& prev) {
+  if (!prev) prev = list->first;
+  if (!prev) return NULL; // empty list, does not exist
+
+  if (prev->key == key) {
+    NODE* n = prev;
+    prev    = NULL;
+    return n;
+  }
+
+  while (prev->next && prev->next->key < key) {
+    prev = prev->next;
+  }
+
+  return prev->next;
+}
+
+
+
+
 /*
  * Finds the node that should go before whatever key we request, whether or not
  * that key is present.
  */
-NODE* find_preceding_from(NODE* prev, size_t key) {
+NODE* find_preceding_from_node(NODE* prev, size_t key) {
   NODE* curr = prev->next;
 
   if (!curr || key <= curr->key) {
   	return prev;
   	
   } else {
-  	return find_preceding_from(curr, key);
+  	return find_preceding_from_node(curr, key);
   }
 }
 
+
+/*
+ * Returns NULL if the key being sought is first in the list or *should* be first in the list but is absent. Otherwise
+ * returns the previous node to where that key is or should be.
+ */
+NODE* find_preceding_from_list(LIST* l, size_t key) {
+  NODE* n = l->first;
+  if (!n || n->key >= key)  return NULL;
+  else                      return find_preceding_from_node(n, key);
+}
+
 /*
  * Finds the node or, if not present, the node that it should follow. NULL
  * indicates no preceding node.
@@ -336,7 +477,7 @@ NODE* find_nearest_from(NODE* prev, size_t key) {
   	return prev;
   }
 
-  f = find_preceding_from(prev, key);
+  f = find_preceding_from_node(prev, key);
 
   if (!f->next) { // key exceeds final node; return final node.
   	return f;