xnd 0.2.0dev6 → 0.2.0dev7

data/ext/ruby_xnd/xnd/python/xnd/_version.py

@@ -0,0 +1 @@
+__version__ = '0.2.0dev3'

data/ext/ruby_xnd/xnd/python/xnd/_xnd.c

@@ -41,6 +41,7 @@
 #include "pyndtypes.h"
 #include "xnd.h"
 #include "util.h"
+#include "overflow.h"
 #include "docstrings.h"
 
 #define XND_MODULE
@@ -77,6 +78,13 @@ seterr_int(ndt_context_t *ctx)
     return -1;
 }
 
+static const ndt_t *
+seterr_ndt(ndt_context_t *ctx)
+{
+    (void)Ndt_SetError(ctx);
+    return NULL;
+}
+
 
 /****************************************************************************/
 /*                                Singletons                                */
@@ -162,7 +170,7 @@ mblock_dealloc(MemoryBlockObject *self)
 }
 
 static MemoryBlockObject *
-mblock_empty(PyObject *type)
+mblock_empty(PyObject *type, uint32_t flags)
 {
     NDT_STATIC_CONTEXT(ctx);
     MemoryBlockObject *self;
@@ -177,7 +185,7 @@ mblock_empty(PyObject *type)
         return NULL;
     }
 
-    self->xnd = xnd_empty_from_type(CONST_NDT(type), XND_OWN_EMBEDDED, &ctx);
+    self->xnd = xnd_empty_from_type(NDT(type), XND_OWN_EMBEDDED|flags, &ctx);
     if (self->xnd == NULL) {
         Py_DECREF(self);
         return (MemoryBlockObject *)seterr(&ctx);
@@ -189,11 +197,11 @@ mblock_empty(PyObject *type)
 }
 
 static MemoryBlockObject *
-mblock_from_typed_value(PyObject *type, PyObject *value)
+mblock_from_typed_value(PyObject *type, PyObject *value, uint32_t flags)
 {
     MemoryBlockObject *self;
 
-    self = mblock_empty(type);
+    self = mblock_empty(type, flags);
     if (self == NULL) {
         return NULL;
     }
@@ -219,7 +227,8 @@ mblock_from_xnd(xnd_t *src)
         return (MemoryBlockObject *)seterr(&ctx);
     }
 
-    type = Ndt_FromType((ndt_t *)x->master.type);
+    type = Ndt_FromType(x->master.type);
+    ndt_decref(x->master.type);
     if (type == NULL) {
         xnd_del(x);
         return NULL;
@@ -242,7 +251,8 @@ static PyObject *
 type_from_buffer(const Py_buffer *view)
 {
     NDT_STATIC_CONTEXT(ctx);
-    ndt_t *t, *type;
+    PyObject *ret;
+    const ndt_t *t, *type;
     int64_t shape, step;
     int64_t i;
 
@@ -269,20 +279,31 @@ type_from_buffer(const Py_buffer *view)
     if (ndt_itemsize(type) != view->itemsize) {
         PyErr_SetString(PyExc_RuntimeError,
             "mismatch between computed itemsize and buffer itemsize");
-        ndt_del(type);
+        ndt_decref(type);
         return NULL;
     }
 
     for (i=view->ndim-1, t=type; i>=0; i--, type=t) {
         shape = view->shape[i];
+
         step = view->strides[i] / view->itemsize;
+        if (step * view->itemsize != view->strides[i]) {
+            PyErr_SetString(PyExc_NotImplementedError,
+                "strides supplied by exporter are not a multiple of itemsize");
+            ndt_decref(type);
+            return NULL;
+        }
+
         t = ndt_fixed_dim(type, shape, step, &ctx);
+        ndt_decref(type);
         if (t == NULL) {
             return seterr(&ctx);
         }
     }
 
-    return Ndt_FromType(t);
+    ret = Ndt_FromType(t);
+    ndt_decref(t);
+    return ret;
 }
 
 static MemoryBlockObject *
@@ -306,15 +327,6 @@ mblock_from_buffer(PyObject *obj)
         return NULL;
     }
 
-    if (!PyBuffer_IsContiguous(self->view, 'A')) {
-        /* Conversion from buf+strides to steps+linear_index is not possible
-           if the start of the original data is missing. */
-        PyErr_SetString(PyExc_NotImplementedError,
-            "conversion from non-contiguous buffers is not implemented");
-        Py_DECREF(self);
-        return NULL;
-    }
-
     self->type = type_from_buffer(self->view);
     if (self->type == NULL) {
         Py_DECREF(self);
@@ -332,16 +344,19 @@ mblock_from_buffer(PyObject *obj)
     self->xnd->master.bitmap.size = 0;
     self->xnd->master.bitmap.next = NULL;
     self->xnd->master.index = 0;
-    self->xnd->master.type = CONST_NDT(self->type);
+    self->xnd->master.type = NDT(self->type);
     self->xnd->master.ptr = self->view->buf;
 
     return self;
 }
 
 static MemoryBlockObject *
-mblock_from_buffer_and_type(PyObject *obj, PyObject *type)
+mblock_from_buffer_and_type(PyObject *obj, PyObject *type, int64_t linear_index,
+                            int64_t bufsize)
 {
+    NDT_STATIC_CONTEXT(ctx);
     MemoryBlockObject *self;
+    const ndt_t *t;
 
     if (!Ndt_Check(type)) {
         PyErr_SetString(PyExc_TypeError, "expected ndt object");
@@ -359,20 +374,27 @@ mblock_from_buffer_and_type(PyObject *obj, PyObject *type)
         return NULL;
     }
 
-    if (PyObject_GetBuffer(obj, self->view, PyBUF_FULL_RO) < 0) {
+    if (PyObject_GetBuffer(obj, self->view, PyBUF_SIMPLE) < 0) {
         Py_DECREF(self);
         return NULL;
     }
 
-    if (!PyBuffer_IsContiguous(self->view, 'A')) {
-        /* Conversion from buf+strides to steps+linear_index is not possible
-           if the start of the original data is missing. */
-        PyErr_SetString(PyExc_NotImplementedError,
-            "conversion from non-contiguous buffers is not implemented");
+    if (self->view->readonly) {
+        PyErr_SetString(PyExc_ValueError, "buffer is readonly");
         Py_DECREF(self);
         return NULL;
     }
 
+    if (bufsize < 0) {
+        bufsize = self->view->len;
+    }
+
+    t = NDT(type);
+    if (xnd_bounds_check(t, linear_index, bufsize, &ctx) < 0) {
+        Py_DECREF(self);
+        return (MemoryBlockObject *)seterr(&ctx);
+    }
+
     Py_INCREF(type);
     self->type = type;
 
@@ -386,8 +408,8 @@ mblock_from_buffer_and_type(PyObject *obj, PyObject *type)
     self->xnd->master.bitmap.data = NULL;
     self->xnd->master.bitmap.size = 0;
     self->xnd->master.bitmap.next = NULL;
-    self->xnd->master.index = 0;
-    self->xnd->master.type = CONST_NDT(self->type);
+    self->xnd->master.index = linear_index;
+    self->xnd->master.type = t;
     self->xnd->master.ptr = self->view->buf;
 
     return self;
@@ -508,6 +530,47 @@ get_uint(PyObject *v, uint64_t max)
     return x;
 }
 
+static int
+union_tag_and_value_from_tuple(uint8_t *tag, PyObject **value, const ndt_t *t, PyObject *tuple)
+{
+    PyObject *name;
+    int64_t i;
+
+    assert(t->tag == Union);
+    assert(PyTuple_Check(tuple));
+
+    if (PyTuple_GET_SIZE(tuple) != 2) {
+        PyErr_SetString(PyExc_ValueError,
+            "unions are represented by a tuple (tag, value), "
+            "where 'tag' is a string");
+        return -1;
+    }
+
+    name = PyTuple_GET_ITEM(tuple, 0);
+    if (!PyUnicode_Check(name)) {
+        PyErr_SetString(PyExc_TypeError,
+            "unions are represented by a tuple (tag, value), "
+            "where 'tag' is a string");
+        return -1;
+    }
+
+    for (i = 0; i < t->Union.ntags; i++) {
+        if (PyUnicode_CompareWithASCIIString(name, t->Union.tags[i]) == 0) {
+            break;
+        }
+    }
+
+    if (i == t->Union.ntags) {
+        PyErr_Format(PyExc_ValueError, "'%s' s not a valid tag", name);
+        return -1;
+    }
+
+    *tag = (uint8_t)i;
+    *value = PyTuple_GET_ITEM(tuple, 1);
+
+    return 0;
+}
+
 static int
 mblock_init(xnd_t * const x, PyObject *v)
 {
@@ -597,6 +660,27 @@ mblock_init(xnd_t * const x, PyObject *v)
         return 0;
     }
 
+    case VarDimElem: {
+        int64_t start, step, shape;
+
+        shape = ndt_var_indices(&start, &step, t, x->index, &ctx);
+        if (shape < 0) {
+            return seterr_int(&ctx);
+        }
+
+        const int64_t i = adjust_index(t->VarDimElem.index, shape, &ctx);
+        if (i < 0) {
+            return seterr_int(&ctx);
+        }
+
+        xnd_t next = xnd_var_dim_next(x, start, step, i);
+        if (mblock_init(&next, v) < 0) {
+            return -1;
+        }
+
+        return 0;
+    }
+
     case Tuple: {
         const int64_t shape = t->Tuple.shape;
         int64_t i;
@@ -670,6 +754,32 @@ mblock_init(xnd_t * const x, PyObject *v)
         return 0;
     }
 
+    case Union: {
+        PyObject *tmp;
+        uint8_t tag;
+
+        if (!PyTuple_Check(v)) {
+            PyErr_Format(PyExc_TypeError,
+                "xnd: expected tuple, not '%.200s'", Py_TYPE(v)->tp_name);
+            return -1;
+        }
+
+        if (union_tag_and_value_from_tuple(&tag, &tmp, t, v) < 0) {
+            return -1;
+        }
+
+        xnd_clear(x, XND_OWN_EMBEDDED);
+        XND_UNION_TAG(x->ptr) = tag;
+
+        xnd_t next = xnd_union_next(x, &ctx);
+        if (next.ptr == NULL) {
+            Py_DECREF(tmp);
+            return seterr_int(&ctx);
+        }
+
+        return mblock_init(&next, tmp);
+    }
+
     case Ref: {
         xnd_t next = xnd_ref_next(x, &ctx);
         if (next.ptr == NULL) {
@@ -806,6 +916,15 @@ mblock_init(xnd_t * const x, PyObject *v)
         return 0;
     }
 
+    case BFloat16: {
+        double tmp = PyFloat_AsDouble(v);
+        if (tmp == -1 && PyErr_Occurred()) {
+            return -1;
+        }
+        xnd_bfloat_pack(x->ptr, tmp);
+        return 0;
+    }
+
     case Float16: {
 #if PY_VERSION_HEX >= 0x03060000
         double tmp = PyFloat_AsDouble(v);
@@ -836,6 +955,16 @@ mblock_init(xnd_t * const x, PyObject *v)
         return _PyFloat_Pack8(tmp, (unsigned char *)x->ptr, le(t->flags));
     }
 
+    case BComplex32: {
+        Py_complex c = PyComplex_AsCComplex(v);
+        if (c.real == -1.0 && PyErr_Occurred()) {
+            return -1;
+        }
+        xnd_bfloat_pack(x->ptr, c.real);
+        xnd_bfloat_pack(x->ptr+2, c.imag);
+        return 0;
+    }
+
     case Complex32: {
 #if PY_VERSION_HEX >= 0x03060000
         Py_complex c = PyComplex_AsCComplex(v);
@@ -1054,6 +1183,43 @@ mblock_init(xnd_t * const x, PyObject *v)
         return 0;
     }
 
+    case Array: {
+        bool overflow = false;
+
+        if (!PyList_Check(v)) {
+            PyErr_Format(PyExc_TypeError,
+                "xnd: expected list, not '%.200s'", Py_TYPE(v)->tp_name);
+            return -1;
+        }
+
+        const Py_ssize_t shape = PyList_GET_SIZE(v);
+        const int64_t size = MULi64(shape, t->Array.itemsize, &overflow);
+        if (overflow) {
+            ndt_err_format(&ctx, NDT_ValueError,
+                "datasize of flexible array is too large");
+            return seterr_int(&ctx);
+        }
+
+        char *data = ndt_aligned_calloc(t->align, size);
+        if (data == NULL) {
+            PyErr_NoMemory();
+            return -1;
+        }
+
+        xnd_clear(x, XND_OWN_EMBEDDED);
+        XND_ARRAY_SHAPE(x->ptr) = shape;
+        XND_ARRAY_DATA(x->ptr) = data;
+
+        for (int64_t i = 0; i < shape; i++) {
+            xnd_t next = xnd_array_next(x, i);
+            if (mblock_init(&next, PyList_GET_ITEM(v, i)) < 0) {
+                return -1;
+            }
+        }
+
+        return 0;
+    }
+
     case Categorical: {
         int64_t k;
 
@@ -1225,20 +1391,59 @@ pyxnd_from_mblock(PyTypeObject *tp, MemoryBlockObject *mblock)
     return (PyObject *)self;
 }
 
+static uint32_t
+device_flags(PyObject *tuple)
+{
+    PyObject *device;
+    PyObject *no;
+
+    if (!PyTuple_Check(tuple) || PyTuple_GET_SIZE(tuple) != 2) {
+        PyErr_SetString(PyExc_TypeError,
+            "device argument must be of the form (device_name, device_no)");
+        return UINT32_MAX;
+    }
+
+    device = PyTuple_GET_ITEM(tuple, 0);
+    if (!PyUnicode_Check(device) ||
+        PyUnicode_CompareWithASCIIString(device, "cuda") != 0) {
+        PyErr_SetString(PyExc_ValueError,
+            "currently only 'cuda' is supported as a device name");
+        return UINT32_MAX;
+    }
+
+    no = PyTuple_GET_ITEM(tuple, 1);
+    if (!PyLong_Check(no) || PyLong_AsLong(no) != -1) {
+        PyErr_SetString(PyExc_ValueError,
+            "currently only 'cuda:managed' is supported as a device");
+        return UINT32_MAX;
+    }
+
+    return XND_CUDA_MANAGED;
+}
+
 static PyObject *
 pyxnd_new(PyTypeObject *tp, PyObject *args, PyObject *kwds)
 {
-    static char *kwlist[] = {"type", "value", NULL};
+    static char *kwlist[] = {"type", "value", "device", NULL};
     PyObject *type = NULL;
     PyObject *value = NULL;
+    PyObject *tuple = Py_None;
     MemoryBlockObject *mblock;
+    uint32_t flags = 0;
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &type,
-        &value)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O", kwlist, &type,
+        &value, &tuple)) {
         return NULL;
     }
 
-    mblock = mblock_from_typed_value(type, value);
+    if (tuple != Py_None) {
+        flags = device_flags(tuple);
+        if (flags == UINT32_MAX) {
+            return NULL;
+        }
+    }
+
+    mblock = mblock_from_typed_value(type, value, flags);
     if (mblock == NULL) {
         return NULL;
     }
@@ -1247,16 +1452,32 @@ pyxnd_new(PyTypeObject *tp, PyObject *args, PyObject *kwds)
 }
 
 static PyObject *
-pyxnd_empty(PyTypeObject *tp, PyObject *type)
+pyxnd_empty(PyTypeObject *tp, PyObject *args, PyObject *kwds)
 {
+    static char *kwlist[] = {"type", "device", NULL};
+    PyObject *type = Py_None;
+    PyObject *tuple = Py_None;
     MemoryBlockObject *mblock;
+    uint32_t flags = 0;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, &type,
+        &tuple)) {
+        return NULL;
+    }
+
+    if (tuple != Py_None) {
+        flags = device_flags(tuple);
+        if (flags == UINT32_MAX) {
+            return NULL;
+        }
+    }
 
     type = Ndt_FromObject(type);
     if (type == NULL) {
         return NULL;
     }
 
-    mblock = mblock_empty(type);
+    mblock = mblock_empty(type, flags);
     Py_DECREF(type);
     if (mblock == NULL) {
         return NULL;
@@ -1290,7 +1511,7 @@ pyxnd_from_buffer_and_type(PyTypeObject *tp, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    mblock = mblock_from_buffer_and_type(obj, type);
+    mblock = mblock_from_buffer_and_type(obj, type, 0, -1);
     if (mblock == NULL) {
         return NULL;
     }
@@ -1402,6 +1623,23 @@ _pyxnd_value(const xnd_t * const x, const int64_t maxshape)
         return lst;
     }
 
+    case VarDimElem: {
+        int64_t start, step, shape;
+
+        shape = ndt_var_indices(&start, &step, t, x->index, &ctx);
+        if (shape < 0) {
+            return seterr(&ctx);
+        }
+
+        const int64_t i = adjust_index(t->VarDimElem.index, shape, &ctx);
+        if (i < 0) {
+            return seterr(&ctx);
+        }
+
+        const xnd_t next = xnd_var_dim_next(x, start, step, i);
+        return _pyxnd_value(&next, maxshape);
+    }
+
     case Tuple: {
         PyObject *tuple, *v;
         int64_t shape, i;
@@ -1485,6 +1723,37 @@ _pyxnd_value(const xnd_t * const x, const int64_t maxshape)
         return dict;
     }
 
+    case Union: {
+        PyObject *tuple, *tag, *v;
+
+        tuple = PyTuple_New(2);
+        if (tuple == NULL) {
+            return NULL;
+        }
+
+        const uint8_t i = XND_UNION_TAG(x->ptr);
+        tag = PyUnicode_FromString(t->Union.tags[i]);
+        if (tag == NULL) {
+            Py_DECREF(tuple);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(tuple, 0, tag);
+
+        const xnd_t next = xnd_union_next(x, &ctx);
+        if (next.ptr == NULL) {
+            return seterr(&ctx);
+        }
+
+        v = _pyxnd_value(&next, maxshape);
+        if (v == NULL) {
+            Py_DECREF(tuple);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(tuple, 1, v);
+
+        return tuple;
+    }
+
     case Ref: {
         const xnd_t next = xnd_ref_next(x, &ctx);
         if (next.ptr == NULL) {
@@ -1570,6 +1839,11 @@ _pyxnd_value(const xnd_t * const x, const int64_t maxshape)
         return PyLong_FromUnsignedLongLong(tmp);
     }
 
+    case BFloat16: {
+        double tmp = xnd_bfloat_unpack(x->ptr);
+        return PyFloat_FromDouble(tmp);
+    }
+
     case Float16: {
 #if PY_VERSION_HEX >= 0x03060000
         double tmp = _PyFloat_Unpack2((unsigned char *)x->ptr, le(t->flags));
@@ -1600,6 +1874,13 @@ _pyxnd_value(const xnd_t * const x, const int64_t maxshape)
         return PyFloat_FromDouble(tmp);
     }
 
+    case BComplex32: {
+        Py_complex c;
+        c.real = xnd_bfloat_unpack(x->ptr);
+        c.imag = xnd_bfloat_unpack(x->ptr+2);
+        return PyComplex_FromCComplex(c);
+    }
+
     case Complex32: {
 #if PY_VERSION_HEX >= 0x03060000
         Py_complex c;
@@ -1681,8 +1962,8 @@ _pyxnd_value(const xnd_t * const x, const int64_t maxshape)
     }
 
     case String: {
-        const char *s = XND_POINTER_DATA(x->ptr);
-        Py_ssize_t size = s ? strlen(s) : 0;
+        const char *s = XND_STRING_DATA(x->ptr);
+        Py_ssize_t size = strlen(s);
 
         return PyUnicode_FromStringAndSize(s, size);
     }
@@ -1694,6 +1975,38 @@ _pyxnd_value(const xnd_t * const x, const int64_t maxshape)
         return bytes_from_string_and_size(s, size);
     }
 
+    case Array: {
+        PyObject *lst, *v;
+        int64_t shape;
+
+        shape = XND_ARRAY_SHAPE(x->ptr);
+        if (shape > maxshape) {
+            shape = maxshape;
+        }
+
+        lst = list_new(shape);
+        if (lst == NULL) {
+            return NULL;
+        }
+
+        for (int64_t i = 0; i < shape; i++) {
+            if (i == maxshape-1) {
+                PyList_SET_ITEM(lst, i, xnd_ellipsis());
+                break;
+            }
+
+            const xnd_t next = xnd_array_next(x, i);
+            v = _pyxnd_value(&next, maxshape);
+            if (v == NULL) {
+                Py_DECREF(lst);
+                return NULL;
+            }
+            PyList_SET_ITEM(lst, i, v);
+        }
+
+        return lst;
+    }
+
     case Categorical: {
         int64_t k;
 
@@ -1760,10 +2073,11 @@ pyxnd_view_move_type(const XndObject *src, xnd_t *x)
     XndObject *view;
     PyObject *type;
 
-    type = Ndt_MoveSubtree(src->type, (ndt_t *)x->type);
+    type = Ndt_FromType(x->type);
     if (type == NULL) {
         return NULL;
     }
+    ndt_decref(x->type);
 
     view = pyxnd_alloc(Py_TYPE(src));
     if (view == NULL) {
@@ -1813,6 +2127,24 @@ pyxnd_len(const xnd_t *x)
         return safe_downcast(shape);
     }
 
+    case VarDimElem: {
+        NDT_STATIC_CONTEXT(ctx);
+        int64_t start, step, shape;
+
+        shape = ndt_var_indices(&start, &step, t, x->index, &ctx);
+        if (shape < 0) {
+            return seterr_int(&ctx);
+        }
+
+        const int64_t i = adjust_index(t->VarDimElem.index, shape, &ctx);
+        if (i < 0) {
+            return seterr_int(&ctx);
+        }
+
+        const xnd_t next = xnd_var_dim_next(x, start, step, i);
+        return pyxnd_len(&next);
+    }
+
     case Tuple: {
         return safe_downcast(t->Tuple.shape);
     }
@@ -1821,6 +2153,20 @@ pyxnd_len(const xnd_t *x)
         return safe_downcast(t->Record.shape);
     }
 
+    case Array: {
+        const int64_t shape = XND_ARRAY_SHAPE(x->ptr);
+        return safe_downcast(shape);
+    }
+
+    case Union: {
+        const xnd_t next = xnd_union_next(x, &ctx);
+        if (next.ptr == NULL) {
+            return seterr_int(&ctx);
+        }
+
+        return pyxnd_len(&next);
+    }
+
     case Ref: {
         const xnd_t next = xnd_ref_next(x, &ctx);
         if (next.ptr == NULL) {
@@ -1932,7 +2278,7 @@ convert_key(xnd_index_t *indices, int *len, PyObject *key)
             }
         }
 
-        *len = size;
+        *len = (int)size;
         return flags;
     }
 
@@ -1962,36 +2308,99 @@ pyxnd_subscript(XndObject *self, PyObject *key)
     return pyxnd_view_move_type(self, &x);
 }
 
-static void
-free_slices(xnd_t *lst, int64_t start, int64_t stop)
-{
-    for (int64_t i = start; i < stop; i++) {
-        ndt_del((ndt_t *)lst[i].type);
-    }
-
-    ndt_free(lst);
-}
-
 static PyObject *
-pyxnd_split(PyObject *self, PyObject *args, PyObject *kwds)
+pyxnd_reshape(PyObject *self, PyObject *args, PyObject *kwds)
 {
-    static char *kwlist[] = {"n", "max_outer", NULL};
+    static char *kwlist[] = {"shape", "order", NULL};
     NDT_STATIC_CONTEXT(ctx);
-    PyObject *max = Py_None;
-    PyObject *nparts;
-    int max_outer = NDT_MAX_DIM;
-    PyObject *res;
-    xnd_t *slices;
-    int64_t n;
-
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, &nparts, &max)) {
+    PyObject *tuple = NULL;
+    PyObject *order = Py_None;
+    int64_t shape[NDT_MAX_DIM];
+    char ord = 'C';
+    Py_ssize_t n;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, &tuple,
+                                     &order)) {
         return NULL;
     }
 
-    n = PyLong_AsLongLong(nparts);
-    if (n == -1 && PyErr_Occurred()) {
-        return NULL;
-    }
+    if (order != Py_None) {
+        const char *c = PyUnicode_AsUTF8(order);
+        if (strlen(c) != 1) {
+            PyErr_SetString(PyExc_TypeError,
+                "'order' argument must be a 'C', 'F' or 'A'");
+            return NULL;
+        }
+        ord = c[0];
+    }
+
+    if (!PyTuple_Check(tuple)) {
+        PyErr_SetString(PyExc_TypeError,
+            "'shape' argument must be a tuple");
+        return NULL;
+    }
+
+    n = PyTuple_GET_SIZE(tuple);
+    if (n > NDT_MAX_DIM) {
+        PyErr_SetString(PyExc_ValueError, "too many dimensions");
+        return NULL;
+    }
+
+    for (int i = 0; i < n; i++) {
+        shape[i] = PyLong_AsLongLong(PyTuple_GET_ITEM(tuple, i));
+        if (shape[i] < 0) {
+            if (PyErr_Occurred()) {
+                return NULL;
+            }
+            PyErr_SetString(PyExc_ValueError, "negative dimension size");
+            return NULL;
+        }
+    }
+
+    xnd_t view = xnd_reshape(XND(self), shape, (int)n, ord, &ctx);
+    if (xnd_err_occurred(&view)) {
+        return seterr(&ctx);
+    }
+
+    return pyxnd_view_move_type((XndObject *)self, &view);
+}
+
+static void
+free_slices(xnd_t *lst, int64_t start, int64_t stop)
+{
+    for (int64_t i = start; i < stop; i++) {
+        ndt_decref(lst[i].type);
+    }
+
+    ndt_free(lst);
+}
+
+static PyObject *
+pyxnd_split(PyObject *self, PyObject *args, PyObject *kwds)
+{
+    static char *kwlist[] = {"n", "max_outer", NULL};
+    NDT_STATIC_CONTEXT(ctx);
+    PyObject *max = Py_None;
+    PyObject *nparts;
+    int max_outer = NDT_MAX_DIM;
+    PyObject *res;
+    xnd_t *slices;
+    int64_t n;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, &nparts, &max)) {
+        return NULL;
+    }
+
+    n = PyLong_AsLongLong(nparts);
+    if (n == -1 && PyErr_Occurred()) {
+        return NULL;
+    }
+
+    if (n < 1 || n > INT32_MAX)  {
+        PyErr_SetString(PyExc_ValueError,
+            "n must be in [1, INT32_MAX]");
+        return NULL;
+    }
 
     if (max != Py_None) {
         long l = PyLong_AsLong(max);
@@ -2011,7 +2420,7 @@ pyxnd_split(PyObject *self, PyObject *args, PyObject *kwds)
         return seterr(&ctx);
     }
 
-    res = PyList_New(n);
+    res = PyList_New((Py_ssize_t)n);
     if (res == NULL) {
         free_slices(slices, 0, n);
         return NULL;
@@ -2038,7 +2447,6 @@ pyxnd_assign(XndObject *self, PyObject *key, PyObject *value)
     NDT_STATIC_CONTEXT(ctx);
     xnd_index_t indices[NDT_MAX_DIM];
     xnd_t x;
-    int free_type = 0;
     int ret, len;
     uint8_t flags;
 
@@ -2057,20 +2465,7 @@ pyxnd_assign(XndObject *self, PyObject *key, PyObject *value)
         return -1;
     }
 
-    if (flags & KEY_SLICE) {
-        x = xnd_multikey(&self->xnd, indices, len, &ctx);
-        if (x.ptr == NULL) {
-            return seterr_int(&ctx);
-        }
-        free_type = 1;
-    }
-    else {
-        x = xnd_subtree(&self->xnd, indices, len, &ctx);
-        if (x.ptr == NULL) {
-            return seterr_int(&ctx);
-        }
-    }
-
+    x = xnd_subscript(&self->xnd, indices, len, &ctx);
     if (x.ptr == NULL) {
         return seterr_int(&ctx);
     }
@@ -2085,10 +2480,7 @@ pyxnd_assign(XndObject *self, PyObject *key, PyObject *value)
         ret = mblock_init(&x, value);
     }
 
-    if (free_type) {
-        ndt_del((ndt_t *)x.type);
-    }
-
+    ndt_decref(x.type);
     return ret;
 }
 
@@ -2108,6 +2500,65 @@ pyxnd_item(XndObject *self, Py_ssize_t index)
     return res;
 }
 
+static PyObject *
+pyxnd_transpose(PyObject *self, PyObject *args, PyObject *kwds)
+{
+    static char *kwlist[] = {"permute", NULL};
+    NDT_STATIC_CONTEXT(ctx);
+    PyObject *permute = Py_None;
+    int p[NDT_MAX_ARGS];
+    const ndt_t *t;
+    xnd_t x;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &permute)) {
+        return NULL;
+    }
+
+    if (permute != Py_None) {
+        if (!PyList_Check(permute) && !PyTuple_Check(permute)) {
+            PyErr_SetString(PyExc_TypeError,
+                "the 'permute' argument must be a list or a tuple");
+            return NULL;
+        }
+
+        const Py_ssize_t len = PySequence_Fast_GET_SIZE(permute);
+
+        if (len > NDT_MAX_ARGS) {
+            PyErr_SetString(PyExc_ValueError, "permutation list too long");
+            return NULL;
+        }
+
+        for (int i = 0; i < len; i++) {
+            int v = PyLong_AsLong(PySequence_Fast_GET_ITEM(permute, i));
+            if (v == -1 && PyErr_Occurred()) {
+                return NULL;
+            }
+
+            if (v < 0 || v > INT_MAX) {
+                PyErr_SetString(PyExc_ValueError,
+                    "permutation index out of bounds");
+                return NULL;
+            }
+
+            p[i] = (int)v;
+        }
+
+        t = ndt_transpose(XND_TYPE(self), p, (int)len, &ctx);
+    }
+    else {
+        t = ndt_transpose(XND_TYPE(self), NULL, 0, &ctx);
+    }
+
+    if (t == NULL) {
+        return seterr(&ctx);
+    }
+
+    x = *XND(self);
+    x.type = t;
+
+    return pyxnd_view_move_type((XndObject *)self, &x);
+}
+
 static PyObject *
 pyxnd_short_value(PyObject *self, PyObject *args, PyObject *kwds)
 {
@@ -2165,6 +2616,13 @@ pyxnd_type(PyObject *self, PyObject *args UNUSED)
     return TYPE_OWNER(self);
 }
 
+static PyObject *
+pyxnd_dtype(PyObject *self, PyObject *args UNUSED)
+{
+    const ndt_t *dtype = ndt_dtype(XND_TYPE(self));
+    return Ndt_FromType(dtype);
+}
+
 static PyObject *
 pyxnd_ndim(PyObject *self, PyObject *args UNUSED)
 {
@@ -2185,13 +2643,273 @@ pyxnd_align(PyObject *self, PyObject *args UNUSED)
     return PyLong_FromUnsignedLong(align);
 }
 
+static PyObject *
+pyxnd_device(XndObject *self, PyObject *args UNUSED)
+{
+    uint32_t flags = self->mblock->xnd->flags;
+
+    if (flags & XND_CUDA_MANAGED) {
+        return PyUnicode_FromString("cuda:managed");
+    }
+
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+pyxnd_copy_contiguous(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+    static char *kwlist[] = {"dtype", NULL};
+    NDT_STATIC_CONTEXT(ctx);
+    XndObject *src = (XndObject *)self;
+    PyObject *dtype = Py_None;
+    PyObject *dest;
+    const ndt_t *t;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O", kwlist, &dtype)) {
+        return NULL;
+    }
+
+    if (dtype != Py_None) {
+        if (!Ndt_Check(dtype)) {
+            PyErr_Format(PyExc_TypeError,
+                "dtype argument must be 'ndt', got '%.200s'",
+                Py_TYPE(dtype)->tp_name);
+            return NULL;
+        }
+        t = ndt_copy_contiguous_dtype(XND_TYPE(src), NDT(dtype), XND_INDEX(src),
+                                      &ctx);
+    }
+    else {
+        t = ndt_copy_contiguous(XND_TYPE(src), XND_INDEX(src), &ctx);
+    }
+
+    if (t == NULL) {
+        return seterr(&ctx);
+    }
+
+    dest = Xnd_EmptyFromType(Py_TYPE(src), t, 0);
+    ndt_decref(t);
+    if (dest == NULL) {
+        return NULL;
+    }
+
+    if (xnd_copy(XND(dest), XND(src), src->mblock->xnd->flags, &ctx) < 0) {
+        Py_DECREF(dest);
+        return seterr(&ctx);
+    }
+
+    return dest;
+}
+
+static PyObject *
+pyxnd_tobytes(PyObject *self, PyObject *args UNUSED)
+{
+    NDT_STATIC_CONTEXT(ctx);
+    XndObject *src = (XndObject *)self;
+    const ndt_t *t = XND_TYPE(self);
+    PyObject *b;
+
+    if (!ndt_is_pointer_free(t)) {
+        PyErr_SetString(PyExc_NotImplementedError,
+            "tobytes() is not implemented for memory blocks with pointers");
+        return NULL;
+    }
+
+    if (ndt_is_optional(t) || ndt_subtree_is_optional(t)) {
+        PyErr_SetString(PyExc_NotImplementedError,
+            "serializing bitmaps is not implemented");
+        return NULL;
+    }
+
+    if (!ndt_is_ndarray(t)) {
+        PyErr_SetString(PyExc_NotImplementedError,
+            "tobytes() is only implemented for ndarrays");
+        return NULL;
+    }
+
+    const bool contiguous = ndt_is_c_contiguous(t) || ndt_is_f_contiguous(t) ||
+                            ndt_is_var_contiguous(t);
+
+    if (contiguous) {
+        ndt_incref(t);
+    }
+    else {
+        t = ndt_copy_contiguous(XND_TYPE(src), XND_INDEX(src), &ctx);
+        if (t == NULL) {
+            return seterr(&ctx);
+        }
+    }
+
+    b = PyBytes_FromStringAndSize(NULL, t->datasize);
+    if (b == NULL) {
+        ndt_decref(t);
+        return NULL;
+    }
+    char *cp = PyBytes_AS_STRING(b);
+
+
+    if (contiguous) {
+         char *ptr = XND(src)->ptr;
+         if (t->ndim != 0) {
+             ptr += XND_INDEX(src) * t->Concrete.FixedDim.itemsize;
+         }
+
+        memcpy(cp, ptr, t->datasize);
+    }
+    else {
+        xnd_t x = xnd_error;
+        x.type = t;
+        x.ptr = cp;
+
+        if (xnd_copy(&x, XND(src), src->mblock->xnd->flags, &ctx) < 0) {
+            Py_DECREF(b);
+            ndt_decref(t);
+            return seterr(&ctx);
+        }
+    }
+
+    ndt_decref(t);
+    return b;
+}
+
+static PyObject *
+_serialize(XndObject *self)
+{
+    NDT_STATIC_CONTEXT(ctx);
+    bool overflow = false;
+    const xnd_t *x = XND(self);
+    const ndt_t *t = XND_TYPE(self);
+    PyObject *result;
+    char *cp, *s;
+    int64_t tlen;
+    int64_t size;
+
+    if (!ndt_is_pointer_free(t)) {
+        PyErr_SetString(PyExc_NotImplementedError,
+            "serializing memory blocks with pointers is not implemented");
+        return NULL;
+    }
+
+    if (ndt_is_optional(t) || ndt_subtree_is_optional(t)) {
+        PyErr_SetString(PyExc_NotImplementedError,
+            "serializing bitmaps is not implemented");
+        return NULL;
+    }
+
+    if (!ndt_is_c_contiguous(t) && !ndt_is_f_contiguous(t) &&
+        !ndt_is_var_contiguous(t)) {
+        PyErr_SetString(PyExc_NotImplementedError,
+            "serializing non-contiguous memory blocks is not implemented");
+        return NULL;
+    }
+
+    tlen = ndt_serialize(&s, t, &ctx);
+    if (tlen < 0) {
+        return seterr(&ctx);
+    }
+
+    size = ADDi64(t->datasize, tlen, &overflow);
+    size = ADDi64(size, 8, &overflow);
+    if (overflow) {
+        PyErr_SetString(PyExc_OverflowError, "too large to serialize");
+        ndt_free(s);
+        return NULL;
+    }
+
+    result = PyBytes_FromStringAndSize(NULL, size);
+    cp = PyBytes_AS_STRING(result);
+
+    char *ptr = x->ptr;
+    if (t->ndim != 0) {
+         ptr = x->ptr + x->index * t->Concrete.FixedDim.itemsize;
+    }
+
+    memcpy(cp, ptr, t->datasize); cp += t->datasize;
+    memcpy(cp, s, tlen); cp += tlen;
+    memcpy(cp, &t->datasize, 8);
+    ndt_free(s);
+
+    return result;
+}
+
+static PyObject *
+pyxnd_serialize(PyObject *self, PyObject *args UNUSED)
+{
+    return _serialize((XndObject *)self);
+}
+
+static PyObject *
+pyxnd_deserialize(PyTypeObject *tp, PyObject *v)
+{
+    NDT_STATIC_CONTEXT(ctx);
+    MemoryBlockObject *mblock;
+    bool overflow = false;
+    int64_t mblock_size;
+
+    if (!PyBytes_Check(v)) {
+        PyErr_Format(PyExc_TypeError,
+            "expected bytes object, not '%.200s'", Py_TYPE(v)->tp_name);
+        return NULL;
+    }
+
+    const int64_t size = PyBytes_GET_SIZE(v);
+    if (size < 8) {
+        goto invalid_format;
+    }
+
+    const char *s = PyBytes_AS_STRING(v);
+    memcpy(&mblock_size, s+size-8, 8);
+    if (mblock_size < 0) {
+        goto invalid_format;
+    }
+
+    const int64_t tmp = ADDi64(mblock_size, 8, &overflow);
+    const int64_t tlen = size-tmp;
+    if (overflow || tlen < 0) {
+        goto invalid_format;
+    }
+
+    const ndt_t *t = ndt_deserialize(s+mblock_size, tlen, &ctx);
+    if (t == NULL) {
+        return seterr(&ctx);
+    }
+
+    if (t->datasize != mblock_size) {
+        goto invalid_format;
+    }
+
+    PyObject *type = Ndt_FromType(t);
+    ndt_decref(t);
+    if (type == NULL) {
+        return NULL;
+    }
+
+    mblock = mblock_empty(type, XND_OWN_EMBEDDED);
+    Py_DECREF(type);
+    if (mblock == NULL) {
+        return NULL;
+    }
+
+    memcpy(mblock->xnd->master.ptr, s, mblock_size);
+
+    return pyxnd_from_mblock(tp, mblock);
+
+
+invalid_format:
+    PyErr_SetString(PyExc_ValueError,
+        "invalid format for xnd deserialization");
+    return NULL;
+}
+
 
 static PyGetSetDef pyxnd_getsets [] =
 {
   { "type", (getter)pyxnd_type, NULL, doc_type, NULL},
+  { "dtype", (getter)pyxnd_dtype, NULL, NULL, NULL},
   { "value", (getter)pyxnd_value, NULL, doc_value, NULL},
   { "align", (getter)pyxnd_align, NULL, doc_align, NULL},
   { "ndim", (getter)pyxnd_ndim, NULL, doc_ndim, NULL},
+  { "device", (getter)pyxnd_device, NULL, NULL, NULL},
   {NULL}
 };
 
@@ -2214,12 +2932,18 @@ static PyMethodDef pyxnd_methods [] =
   /* Methods */
   { "short_value", (PyCFunction)pyxnd_short_value, METH_VARARGS|METH_KEYWORDS, doc_short_value },
   { "strict_equal", (PyCFunction)pyxnd_strict_equal, METH_O, NULL },
+  { "copy_contiguous", (PyCFunction)pyxnd_copy_contiguous, METH_VARARGS|METH_KEYWORDS, NULL },
   { "split", (PyCFunction)pyxnd_split, METH_VARARGS|METH_KEYWORDS, NULL },
+  { "transpose", (PyCFunction)pyxnd_transpose, METH_VARARGS|METH_KEYWORDS, NULL },
+  { "tobytes", (PyCFunction)pyxnd_tobytes, METH_NOARGS, NULL },
+  { "_reshape", (PyCFunction)pyxnd_reshape, METH_VARARGS|METH_KEYWORDS, NULL },
+  { "_serialize", (PyCFunction)pyxnd_serialize, METH_NOARGS, NULL },
 
   /* Class methods */
-  { "empty", (PyCFunction)pyxnd_empty, METH_O|METH_CLASS, doc_empty },
+  { "empty", (PyCFunction)pyxnd_empty, METH_VARARGS|METH_KEYWORDS|METH_CLASS, doc_empty },
   { "from_buffer", (PyCFunction)pyxnd_from_buffer, METH_O|METH_CLASS, doc_from_buffer },
-  { "_unsafe_from_data", (PyCFunction)pyxnd_from_buffer_and_type, METH_VARARGS|METH_KEYWORDS|METH_CLASS, NULL },
+  { "from_buffer_and_type", (PyCFunction)pyxnd_from_buffer_and_type, METH_VARARGS|METH_KEYWORDS|METH_CLASS, NULL },
+  { "deserialize", (PyCFunction)pyxnd_deserialize, METH_O|METH_CLASS, NULL },
 
   { NULL, NULL, 1 }
 };
@@ -2455,32 +3179,856 @@ static PyTypeObject Xnd_Type =
 
 
 /****************************************************************************/
-/*                                   C-API                                  */
+/*                               Type inference                             */
 /****************************************************************************/
 
-static void **xnd_api[XND_MAX_API];
+/**********************************************************************/
+/*    Extract data and shapes from a value (possibly a nested list)   */
+/**********************************************************************/
 
+#undef max
 static int
-Xnd_CheckExact(const PyObject *v)
+max(int x, int y)
 {
-    return Py_TYPE(v) == &Xnd_Type;
+    return x >= y ? x : y;
 }
 
+#undef min
 static int
-Xnd_Check(const PyObject *v)
+min(int x, int y)
 {
-    return PyObject_TypeCheck(v, &Xnd_Type);
+    return x <= y ? x : y;
 }
 
-static const xnd_t *
-CONST_XND(const PyObject *v)
+#define XND_NONE 0x0001U
+#define XND_DATA 0x0002U
+#define XND_LIST 0x0004U
+
+static inline int
+check_level(int level)
 {
-    assert(Xnd_Check(v));
-    return &((XndObject *)v)->xnd;
+    if (level >= NDT_MAX_DIM) {
+        PyErr_Format(PyExc_ValueError,
+            "too many dimensions, max %d", NDT_MAX_DIM);
+        return -1;
+    }
+
+    return 0;
+}
+
+static int
+search(int level, PyObject *v, PyObject *data, PyObject *acc[NDT_MAX_DIM],
+       int *min_level, int *max_level)
+{
+    PyObject *shape;
+    PyObject *item;
+    Py_ssize_t len, i;
+    int next_level;
+    int ret;
+
+    if (PyList_Check(v)) {
+        if (check_level(level) < 0) {
+            return -1;
+        }
+
+        len = PyList_GET_SIZE(v);
+        shape = PyLong_FromSsize_t(len);
+        if (shape == NULL) {
+            return -1;
+        }
+
+        ret = PyList_Append(acc[level], shape);
+        Py_DECREF(shape);
+        if (ret < 0) {
+            return -1;
+        }
+
+        next_level = level + 1;
+        *max_level = max(next_level, *max_level);
+
+        if (len == 0) {
+            *min_level = min(next_level, *min_level);
+        }
+        else {
+            uint32_t types = 0;
+            for (i = 0; i < len; i++) {
+                item = PyList_GET_ITEM(v, i);
+                if (item == Py_None) {
+                    types |= XND_NONE;
+                }
+                else if (PyList_Check(item)) {
+                    types |= XND_LIST;
+                }
+                else {
+                    types |= XND_DATA;
+                }
+            }
+
+            if (!(types & XND_LIST)) {
+                for (i = 0; i < len; i++) {
+                    item = PyList_GET_ITEM(v, i);
+                    if (PyList_Append(data, item) < 0) {
+                        return -1;
+                    }
+                }
+                *min_level = min(next_level, *min_level);
+            }
+            else if (!(types & XND_DATA)) {
+                if (check_level(next_level) < 0) {
+                    return -1;
+                }
+
+                for (i = 0; i < len; i++) {
+                    item = PyList_GET_ITEM(v, i);
+                    if (item == Py_None) {
+                        if (PyList_Append(acc[next_level], item) < 0) {
+                            return -1;
+                        }
+                    }
+                    else {
+                        if (search(next_level, item, data, acc,
+                                   min_level, max_level) < 0) {
+                            return -1;
+                        }
+                    }
+                }
+            }
+            else {
+                PyErr_Format(PyExc_ValueError,
+                    "lists that contain both data and lists cannot be typed");
+                return -1;
+            }
+        }
+    }
+    else {
+        if (PyList_Append(data, v) < 0) {
+            return -1;
+        }
+        *min_level = min(level, *min_level);
+    }
+
+    return 0;
 }
 
 static PyObject *
-Xnd_EmptyFromType(PyTypeObject *tp, ndt_t *t)
+data_shapes(PyObject *m UNUSED, PyObject *v)
+{
+    PyObject *acc[NDT_MAX_DIM] = {NULL};
+    PyObject *data = NULL;
+    PyObject *shapes = NULL;
+    PyObject *tuple = NULL;
+    int min_level = NDT_MAX_DIM;
+    int max_level = 0;
+    int i, k;
+
+    data = PyList_New(0);
+    if (data == NULL) {
+        return NULL;
+    }
+
+    for (i = 0; i < NDT_MAX_DIM; i++) {
+        acc[i] = PyList_New(0);
+        if (acc[i] == NULL) {
+            goto error;
+        }
+    }
+
+    if (search(0, v, data, acc, &min_level, &max_level) < 0) {
+        goto error;
+    }
+
+    if (min_level != max_level) {
+        PyErr_Format(PyExc_ValueError,
+            "unbalanced nested list: min depth: %d max depth: %d",
+            min_level, max_level);
+        goto error;
+    }
+
+    shapes = PyList_New(max_level);
+    if (shapes == NULL) {
+        goto error;
+    }
+
+    for (i=0, k=max_level-1; i < max_level; i++, k--) {
+        PyList_SET_ITEM(shapes, i, acc[k]);
+    }
+
+    for (; i < NDT_MAX_DIM; i++) {
+        Py_DECREF(acc[i]);
+    }
+
+    tuple = PyTuple_New(2);
+    if (tuple == NULL) {
+        Py_DECREF(data);
+        Py_DECREF(shapes);
+        return NULL;
+    }
+    PyTuple_SET_ITEM(tuple, 0, data);
+    PyTuple_SET_ITEM(tuple, 1, shapes);
+
+    return tuple;
+
+error:
+    Py_XDECREF(data);
+    for (i = 0; i < NDT_MAX_DIM; i++) {
+        Py_XDECREF(acc[i]);
+    }
+    return NULL;
+}
+
+
+/**********************************************************************/
+/*    Construct fixed or var dimensions from a list of shape lists    */
+/**********************************************************************/
+
+static bool
+require_var(PyObject *lst)
+{
+    PyObject *shapes;
+    PyObject *v, *w;
+    Py_ssize_t len;
+    Py_ssize_t i, k;
+
+    assert(PyList_Check(lst));
+
+    for (i = 0; i < PyList_GET_SIZE(lst); i++) {
+        shapes = PyList_GET_ITEM(lst, i);
+        assert(PyList_Check(shapes));
+
+        len = PyList_GET_SIZE(shapes);
+        if (len == 0) {
+            continue;
+        }
+
+        v = PyList_GET_ITEM(shapes, 0);
+        if (v == Py_None) {
+            return true;
+        }
+        assert(PyLong_Check(v));
+
+        for (k = 1; k < PyList_GET_SIZE(shapes); k++) {
+            w = PyList_GET_ITEM(shapes, k);
+            if (w == Py_None) {
+                return true;
+            }
+            assert(PyLong_Check(w));
+
+            if (long_compare((PyLongObject *)v, (PyLongObject *)w) != 0) {
+                return true;
+            }
+        }
+    }
+
+    return false;
+}
+
+static const ndt_t *
+fixed_from_shapes(PyObject *lst, const ndt_t *type)
+{
+    NDT_STATIC_CONTEXT(ctx);
+    PyObject *shapes;
+    PyObject *v;
+    Py_ssize_t len;
+    Py_ssize_t shape;
+    Py_ssize_t i;
+    const ndt_t *t;
+
+    assert(PyList_Check(lst));
+
+    ndt_incref(type);
+
+    for (i=0, t=type; i < PyList_GET_SIZE(lst); i++, type=t) {
+        shapes = PyList_GET_ITEM(lst, i);
+        assert(PyList_Check(shapes));
+
+        len = PyList_GET_SIZE(shapes);
+
+        if (len == 0) {
+            shape = 0;
+        }
+        else {
+            v = PyList_GET_ITEM(shapes, 0);
+            shape = PyLong_AsSsize_t(v);
+            if (shape < 0) {
+                ndt_decref(t);
+                return NULL;
+            }
+        }
+
+        t = ndt_fixed_dim(type, shape, INT64_MAX, &ctx);
+        ndt_decref(type);
+        if (t == NULL) {
+            return seterr_ndt(&ctx);
+        }
+    }
+
+    return t;
+}
+
+static const ndt_t *
+var_from_shapes(PyObject *lst, const ndt_t *dtype)
+{
+    NDT_STATIC_CONTEXT(ctx);
+    bool overflow = false;
+    const ndt_t *t;
+    ndt_offsets_t *offsets;
+    int32_t *ptr;
+    int64_t sum;
+    Py_ssize_t len, slen;
+    Py_ssize_t shape;
+    Py_ssize_t i, k;
+    bool opt;
+
+    assert(PyList_Check(lst));
+    len = PyList_GET_SIZE(lst);
+
+    ndt_incref(dtype);
+
+    for (i=0, t=dtype; i < len; i++, dtype=t) {
+        PyObject *shapes = PyList_GET_ITEM(lst, i);
+        assert(PyList_Check(shapes));
+        slen = PyList_GET_SIZE(shapes);
+
+        if (slen+1 > INT32_MAX) {
+            PyErr_SetString(PyExc_ValueError,
+                "variable dimension is too large");
+            return NULL;
+        }
+
+        offsets = ndt_offsets_new((int32_t)(slen+1), &ctx);
+        if (offsets == NULL) {
+            return seterr_ndt(&ctx);
+        }
+
+        ptr = (int32_t *)offsets->v;
+        sum = 0;
+        ptr[0] = 0;
+        opt = false;
+
+        for (k = 0; k < slen; k++) {
+            PyObject *v = PyList_GET_ITEM(shapes, k);
+
+            if (v == Py_None) {
+                shape = 0;
+                opt = true;
+            }
+            else {
+                shape = PyLong_AsSsize_t(v);
+                if (shape < 0) {
+                    ndt_decref_offsets(offsets);
+                    return NULL;
+                }
+            }
+
+            sum = ADDi64(sum, shape, &overflow);
+            if (overflow || sum > INT32_MAX) {
+                PyErr_SetString(PyExc_ValueError,
+                    "variable dimension is too large");
+                ndt_decref_offsets(offsets);
+                return NULL;
+            }
+
+            ptr[k+1] = (int32_t)sum;
+        }
+
+        t = ndt_var_dim(dtype, offsets, 0, NULL, opt, &ctx);
+
+        ndt_decref(dtype);
+        ndt_decref_offsets(offsets);
+
+        if (t == NULL) {
+            return seterr_ndt(&ctx);
+        }
+    }
+
+    return t;
+}
+
+
+/**********************************************************************/
+/*               Infer the dtype from a flat list of data             */
+/**********************************************************************/
+
+static const ndt_t *typeof(PyObject *v, bool replace_any, bool shortcut);
+
+
+#define XND_BOOL       0x0001U
+#define XND_FLOAT64    0x0002U
+#define XND_COMPLEX128 0x0004U
+#define XND_INT64      0x0008U
+#define XND_STRING     0x0010U
+#define XND_BYTES      0x0020U
+#define XND_OTHER      0x0040U
+
+static inline uint32_t
+fast_dtypes(bool *opt, const PyObject *data)
+{
+    uint32_t dtypes = 0;
+
+    assert(PyList_Check(data));
+
+    for (Py_ssize_t i = 0; i < PyList_GET_SIZE(data); i++) {
+        PyObject *v = PyList_GET_ITEM(data, i);
+
+        if (v == Py_None) {
+            *opt = true;
+        }
+        else if (PyBool_Check(v)) {
+            dtypes |= XND_BOOL;
+        }
+        else if (PyFloat_Check(v)) {
+            dtypes |= XND_FLOAT64;
+        }
+        else if (PyComplex_Check(v)) {
+            dtypes |= XND_COMPLEX128;
+        }
+        else if (PyLong_Check(v)) {
+            dtypes |= XND_INT64;
+        }
+        else if (PyUnicode_Check(v)) {
+            dtypes |= XND_STRING;
+        }
+        else if (PyBytes_Check(v)) {
+            dtypes |= XND_BYTES;
+        }
+        else {
+            dtypes |= XND_OTHER;
+        }
+    }
+
+    if (dtypes == 0) {
+        dtypes |= XND_FLOAT64;
+    }
+
+    return dtypes;
+}
+
+static const ndt_t *
+unify_dtypes(const PyObject *data, bool shortcut)
+{
+    NDT_STATIC_CONTEXT(ctx);
+    PyObject *v;
+    const ndt_t *dtype;
+    const ndt_t *t, *u;
+    Py_ssize_t i;
+
+    if (!PyList_Check(data) || PyList_GET_SIZE(data) == 0) {
+        PyErr_Format(PyExc_RuntimeError,
+            "internal error: unify_dtypes expects non-empty list");
+        return NULL;
+    }
+
+    v = PyList_GET_ITEM(data, 0);
+    dtype = typeof(v, false, shortcut);
+    if (dtype == NULL) {
+        return NULL;
+    }
+
+    for (i = 1; i < PyList_GET_SIZE(data); i++) {
+         v = PyList_GET_ITEM(data, i);
+         t = typeof(v, false, shortcut);
+         if (t == NULL) {
+             ndt_decref(dtype);
+             return NULL;
+         }
+
+         if (ndt_equal(t, dtype)) {
+             ndt_decref(t);
+         }
+         else {
+             u = ndt_unify(t, dtype, &ctx);
+             ndt_decref(t);
+             ndt_decref(dtype);
+             if (u == NULL) {
+                 return seterr_ndt(&ctx);
+             }
+             dtype = u;
+        }
+
+        if (shortcut && ndt_is_concrete(dtype)) {
+            break;
+        }
+    }
+
+    if (ndt_is_abstract(dtype)) {
+        const ndt_t *u = ndt_unify_replace_any(dtype, dtype, &ctx);
+        ndt_decref(dtype);
+        if (u == NULL) {
+            return seterr_ndt(&ctx);
+        }
+        dtype = u;
+    }
+
+    return dtype;
+}
+
+const ndt_t *
+typeof_data(const PyObject *data, bool shortcut)
+{
+    NDT_STATIC_CONTEXT(ctx);
+    uint16_opt_t align = {None, 0};
+    const ndt_t *dtype = NULL;
+    bool opt = false;
+    uint32_t dtypes;
+
+    dtypes = fast_dtypes(&opt, data);
+
+    switch (dtypes) {
+    case XND_BOOL:
+        dtype = ndt_primitive(Bool, opt, &ctx);
+        break;
+    case XND_FLOAT64:
+        dtype = ndt_primitive(Float64, opt, &ctx);
+        break;
+    case XND_COMPLEX128:
+        dtype = ndt_primitive(Complex128, opt, &ctx);
+        break;
+    case XND_INT64:
+        dtype = ndt_primitive(Int64, opt, &ctx);
+        break;
+    case XND_STRING:
+        dtype = ndt_string(opt, &ctx);
+        break;
+    case XND_BYTES:
+        dtype = ndt_bytes(align, opt, &ctx);
+        break;
+    default:
+        dtype = unify_dtypes(data, shortcut);
+        if (dtype == NULL) {
+            return NULL;
+        }
+        break;
+    }
+
+    if (dtype == NULL) {
+        return seterr_ndt(&ctx);
+    }
+
+    return dtype;
+}
+
+
+/**********************************************************************/
+/*                          Main type inference                       */
+/**********************************************************************/
+
+static const ndt_t *
+typeof_list_top(PyObject *v, const ndt_t *dtype)
+{
+    PyObject *tuple;
+    PyObject *shapes;
+    const ndt_t *t;
+
+    assert(PyList_Check(v));
+
+    tuple = data_shapes(NULL, v);
+    if (tuple == NULL) {
+        return NULL;
+    }
+    shapes = PyTuple_GET_ITEM(tuple, 1);
+
+    if (require_var(shapes)) {
+        t = var_from_shapes(shapes, dtype);
+    }
+    else {
+        t = fixed_from_shapes(shapes, dtype);
+    }
+
+    Py_DECREF(tuple);
+    return t;
+}
+
+static const ndt_t *
+typeof_list(PyObject *v, bool shortcut)
+{
+    PyObject *tuple;
+    PyObject *data;
+    PyObject *shapes;
+    const ndt_t *t, *dtype;
+
+    assert(PyList_Check(v));
+
+    tuple = data_shapes(NULL, v);
+    if (tuple == NULL) {
+        return NULL;
+    }
+    data = PyTuple_GET_ITEM(tuple, 0);
+    shapes = PyTuple_GET_ITEM(tuple, 1);
+
+    dtype = typeof_data(data, shortcut);
+    if (dtype == NULL) {
+        Py_DECREF(tuple);
+        return NULL;
+    }
+
+    if (require_var(shapes)) {
+        t = var_from_shapes(shapes, dtype);
+    }
+    else {
+        t = fixed_from_shapes(shapes, dtype);
+    }
+
+    ndt_decref(dtype);
+    Py_DECREF(tuple);
+    return t;
+}
+
+static const ndt_t *
+typeof_tuple(PyObject *v, bool replace_any, bool shortcut)
+{
+    NDT_STATIC_CONTEXT(ctx);
+    uint16_opt_t none = {None, 0};
+    ndt_field_t *fields;
+    const ndt_t *t;
+    int64_t shape;
+    int64_t i;
+
+    assert(PyTuple_Check(v));
+
+    shape = PyTuple_GET_SIZE(v);
+    if (shape == 0) {
+        t = ndt_tuple(Nonvariadic, NULL, 0, none, none, false, &ctx);
+        return t == NULL ? seterr_ndt(&ctx) : t;
+    }
+
+    fields = ndt_calloc(shape, sizeof *fields);
+    if (fields == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+
+    for (i = 0; i < shape; i++) {
+        t = typeof(PyTuple_GET_ITEM(v, i), replace_any, shortcut);
+        if (t == NULL) {
+            ndt_field_array_del(fields, i);
+            return NULL;
+        }
+
+        fields[i].access = t->access;
+        fields[i].name = NULL;
+        fields[i].type = t;
+        if (fields[i].access == Concrete) {
+            fields[i].Concrete.align = t->align;
+            fields[i].Concrete.explicit_align = false;
+            fields[i].Concrete.pad = UINT16_MAX;
+            fields[i].Concrete.explicit_pad = false;
+        }
+    }
+
+    t = ndt_tuple(Nonvariadic, fields, shape, none, none, false, &ctx);
+    ndt_field_array_del(fields, shape);
+    return t == NULL ? seterr_ndt(&ctx) : t;
+}
+
+static const ndt_t *
+typeof_dict(PyObject *v, bool replace_any, bool shortcut)
+{
+    NDT_STATIC_CONTEXT(ctx);
+    uint16_opt_t none = {None, 0};
+    PyObject *keys = NULL;
+    PyObject *values = NULL;
+    ndt_field_t *fields;
+    const ndt_t *t;
+    const char *cp;
+    char *name;
+    int64_t shape;
+    int64_t i;
+
+    assert(PyDict_Check(v));
+
+    shape = PyMapping_Size(v);
+    if (shape == 0) {
+        t = ndt_record(Nonvariadic, NULL, 0, none, none, false, &ctx);
+        return t == NULL ? seterr_ndt(&ctx) : t;
+    }
+
+    keys = PyMapping_Keys(v);
+    if (keys == NULL) {
+        return NULL;
+    }
+
+    values = PyMapping_Values(v);
+    if (values == NULL) {
+        Py_DECREF(keys);
+        return NULL;
+    }
+
+    fields = ndt_calloc(shape, sizeof *fields);
+    if (fields == NULL) {
+        Py_DECREF(keys);
+        Py_DECREF(values);
+        PyErr_NoMemory();
+        return NULL;
+    }
+
+    for (i = 0; i < shape; i++) {
+        t = typeof(PyList_GET_ITEM(values, i), replace_any, shortcut);
+        if (t == NULL) {
+            ndt_field_array_del(fields, i);
+            Py_DECREF(keys);
+            Py_DECREF(values);
+            return NULL;
+        }
+
+        cp = PyUnicode_AsUTF8(PyList_GET_ITEM(keys, i));
+        if (cp == NULL) {
+            ndt_field_array_del(fields, i);
+            ndt_decref(t);
+            Py_DECREF(keys);
+            Py_DECREF(values);
+            return NULL;
+        }
+
+        name = ndt_strdup(cp, &ctx);
+        if (name == NULL) {
+            ndt_field_array_del(fields, i);
+            ndt_decref(t);
+            Py_DECREF(keys);
+            Py_DECREF(values);
+            return seterr_ndt(&ctx);
+        }
+
+        fields[i].access = t->access;
+        fields[i].name = name;
+        fields[i].type = t;
+        if (fields[i].access == Concrete) {
+            fields[i].Concrete.align = t->align;
+            fields[i].Concrete.explicit_align = false;
+            fields[i].Concrete.pad = UINT16_MAX;
+            fields[i].Concrete.explicit_pad = false;
+        }
+    }
+
+    Py_DECREF(keys);
+    Py_DECREF(values);
+
+    t = ndt_record(Nonvariadic, fields, shape, none, none, false, &ctx);
+    ndt_field_array_del(fields, shape);
+    return t == NULL ? seterr_ndt(&ctx) : t;
+}
+
+static const ndt_t *
+typeof(PyObject *v, bool replace_any, bool shortcut)
+{
+    NDT_STATIC_CONTEXT(ctx);
+    const ndt_t *t;
+
+    if (PyList_Check(v)) {
+        return typeof_list(v, shortcut);
+    }
+    if (PyTuple_Check(v)) {
+        return typeof_tuple(v, replace_any, shortcut);
+    }
+    if (PyDict_Check(v)) {
+        return typeof_dict(v, replace_any, shortcut);
+    }
+
+    if (PyBool_Check(v)) {
+        t = ndt_primitive(Bool, 0, &ctx);
+    }
+    else if (PyFloat_Check(v)) {
+        t = ndt_primitive(Float64, 0, &ctx);
+    }
+    else if (PyComplex_Check(v)) {
+        t = ndt_primitive(Complex128, 0, &ctx);
+    }
+    else if (PyLong_Check(v)) {
+        t = ndt_primitive(Int64, 0, &ctx);
+    }
+    else if (PyUnicode_Check(v)) {
+        t = ndt_string(false, &ctx);
+    }
+    else if (PyBytes_Check(v)) {
+        uint16_opt_t align = {None, 0};
+        t = ndt_bytes(align, false, &ctx);
+    }
+    else if (v == Py_None) {
+        if (replace_any) {
+            t = ndt_primitive(Float64, NDT_OPTION, &ctx);
+        }
+        else {
+            t = ndt_any_kind(true, &ctx);
+        }
+    }
+    else {
+        PyErr_SetString(PyExc_ValueError, "type inference failed");
+        return NULL;
+    }
+
+    return t == NULL ? seterr_ndt(&ctx) : t;
+}
+
+static PyObject *
+xnd_typeof(PyObject *m UNUSED, PyObject *args, PyObject *kwds)
+{
+    static char *kwlist[] = {"value", "dtype", "shortcut", NULL};
+    PyObject *value = NULL;
+    PyObject *dtype = Py_None;
+    PyObject *ret;
+    const ndt_t *t;
+    int shortcut = 0;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Op", kwlist, &value,
+        &dtype, &shortcut)) {
+        return NULL;
+    }
+
+    if (dtype != Py_None) {
+        if (!Ndt_Check(dtype)) {
+            PyErr_Format(PyExc_ValueError, "dtype argument must be ndt");
+            return NULL;
+        }
+
+        if (PyList_Check(value)) {
+            t = typeof_list_top(value, NDT(dtype));
+        }
+        else {
+            t = NDT(dtype);
+            ndt_incref(t);
+        }
+    }
+    else {
+        t = typeof(value, true, (bool)shortcut);
+    }
+
+    if (t == NULL) {
+        return NULL;
+    }
+
+    ret = Ndt_FromType(t);
+
+    ndt_decref(t);
+    return ret;
+}
+ 
+
+/****************************************************************************/
+/*                                   C-API                                  */
+/****************************************************************************/
+
+static void **xnd_api[XND_MAX_API];
+
+static int
+Xnd_CheckExact(const PyObject *v)
+{
+    return Py_TYPE(v) == &Xnd_Type;
+}
+
+static int
+Xnd_Check(const PyObject *v)
+{
+    return PyObject_TypeCheck(v, &Xnd_Type);
+}
+
+static const xnd_t *
+CONST_XND(const PyObject *v)
+{
+    assert(Xnd_Check(v));
+    return &((XndObject *)v)->xnd;
+}
+
+static PyObject *
+Xnd_EmptyFromType(PyTypeObject *tp, const ndt_t *t, uint32_t flags)
 {
     MemoryBlockObject *mblock;
     PyObject *type;
@@ -2490,7 +4038,7 @@ Xnd_EmptyFromType(PyTypeObject *tp, ndt_t *t)
         return NULL;
     }
 
-    mblock = mblock_empty(type);
+    mblock = mblock_empty(type, flags);
     Py_DECREF(type);
     if (mblock == NULL) {
         return NULL;
@@ -2508,14 +4056,15 @@ Xnd_ViewMoveNdt(const PyObject *v, ndt_t *t)
 
     if (!Xnd_Check(v)) {
         PyErr_SetString(PyExc_TypeError, "expected xnd object");
-        ndt_del(t);
+        ndt_decref(t);
         return NULL;
     }
 
-    type = Ndt_MoveSubtree(src->type, t);
+    type = Ndt_FromType(t);
     if (type == NULL) {
         return NULL;
     }
+    ndt_decref(t);
 
     view = pyxnd_alloc(Py_TYPE(src));
     if (view == NULL) {
@@ -2571,7 +4120,7 @@ Xnd_FromXndMoveType(const PyObject *xnd, xnd_t *x)
     if (!Xnd_Check(xnd)) {
         PyErr_SetString(PyExc_TypeError,
             "Xnd_FromXndMoveType() called on non-xnd object");
-        ndt_del((ndt_t *)x->type);
+        ndt_decref(x->type);
         return NULL;
     }
 
@@ -2700,7 +4249,7 @@ _test_view_new(PyObject *module UNUSED, PyObject *args UNUSED)
     NDT_STATIC_CONTEXT(ctx);
     xnd_view_t x = xnd_view_error;
     double *d;
-    ndt_t *t;
+    const ndt_t *t;
     char *ptr;
 
     t = ndt_from_string("3 * float64", &ctx);
@@ -2710,7 +4259,7 @@ _test_view_new(PyObject *module UNUSED, PyObject *args UNUSED)
 
     ptr = ndt_aligned_calloc(8, 3 * sizeof(double));
     if (ptr == NULL) {
-        ndt_del(t);
+        ndt_decref(t);
         (void)ndt_memory_error(&ctx);
         return seterr(&ctx);
     }
@@ -2736,6 +4285,8 @@ _test_view_new(PyObject *module UNUSED, PyObject *args UNUSED)
 
 static PyMethodDef _xnd_methods [] =
 {
+  { "data_shapes", (PyCFunction)data_shapes, METH_O, NULL},
+  { "_typeof", (PyCFunction)xnd_typeof, METH_VARARGS|METH_KEYWORDS, NULL},
   { "_test_view_subscript", (PyCFunction)_test_view_subscript, METH_VARARGS|METH_KEYWORDS, NULL},
   { "_test_view_new", (PyCFunction)_test_view_new, METH_NOARGS, NULL},
   { NULL, NULL, 1, NULL }