ndtypes 0.2.0dev5 → 0.2.0dev6

data/ext/ruby_ndtypes/ndtypes/libndtypes/compat/bplexer.h

@@ -727,7 +727,7 @@ extern int yylex \
 #undef yyTABLES_NAME
 #endif
 
-#line 176 "bplexer.l"
+#line 177 "bplexer.l"
 
 
 #line 733 "bplexer.h"

data/ext/ruby_ndtypes/ndtypes/libndtypes/compat/bplexer.l

@@ -136,9 +136,10 @@ yycolumn = 1;
 "e"        { yylval->uchar = 'e'; return DTYPE; }
 "f"        { yylval->uchar = 'f'; return DTYPE; }
 "d"        { yylval->uchar = 'd'; return DTYPE; }
-"E"        { yylval->uchar = 'E'; return DTYPE; }
-"F"        { yylval->uchar = 'F'; return DTYPE; }
-"D"        { yylval->uchar = 'D'; return DTYPE; }
+
+"Ze"       { yylval->uchar = 'E'; return DTYPE; }
+"Zf"       { yylval->uchar = 'F'; return DTYPE; }
+"Zd"       { yylval->uchar = 'D'; return DTYPE; }
 
 "x"        { return PAD; }
 "T"        { return RECORD; }

data/ext/ruby_ndtypes/ndtypes/libndtypes/compat/export.c

@@ -260,23 +260,24 @@ format(buf_t *buf, const ndt_t *t, ndt_context_t *ctx)
         case Float32: return primitive(buf, t, "f", ctx);
         case Float64: return primitive(buf, t, "d", ctx);
 
-        case Complex32: return primitive(buf, t, "E", ctx);
-        case Complex64: return primitive(buf, t, "F", ctx);
-        case Complex128: return primitive(buf, t, "D", ctx);
+        case Complex32: return primitive(buf, t, "Ze", ctx);
+        case Complex64: return primitive(buf, t, "Zf", ctx);
+        case Complex128: return primitive(buf, t, "Zd", ctx);
 
         case Char:
             if (t->Char.encoding == Ascii) return ndt_snprintf(ctx, buf, "c");
             /* fall through */
 
         case Module: case Function:
-        case VarDim: case SymbolicDim: case EllipsisDim:
-        case Ref: case Constr: case Nominal:
+        case VarDim: case VarDimElem: case SymbolicDim: case EllipsisDim:
+        case Union: case Ref: case Constr: case Nominal:
         case Categorical:
-        case FixedString: case String: case Bytes:
+        case FixedString: case String: case Bytes: case Array:
         case Typevar:
         case AnyKind: case ScalarKind:
         case SignedKind: case UnsignedKind:
-        case FloatKind: case ComplexKind:
+        case FloatKind: case BFloat16:
+        case ComplexKind: case BComplex32:
         case FixedStringKind: case FixedBytesKind:
             ndt_err_format(ctx, NDT_ValueError,
                 "type is not supported by the buffer protocol");

data/ext/ruby_ndtypes/ndtypes/libndtypes/compat/import.c

@@ -49,14 +49,14 @@ int ndt_bpdebug = 1;
 jmp_buf ndt_bp_lexerror;
 
 
-ndt_t *
+const ndt_t *
 ndt_from_bpformat(const char *input, ndt_context_t *ctx)
 {
     volatile yyscan_t scanner = NULL;
     volatile YY_BUFFER_STATE state = NULL;
     char *buffer;
     size_t size;
-    ndt_t *ast = NULL;
+    const ndt_t *ast = NULL;
     int ret;
 
     size = strlen(input);

data/ext/ruby_ndtypes/ndtypes/libndtypes/copy.c

@@ -33,8 +33,11 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
 #include <assert.h>
 #include "ndtypes.h"
+#include "overflow.h"
 
 
 static inline void
@@ -48,22 +51,59 @@ copy_common(ndt_t *u, const ndt_t *t)
     u->align = t->align;
 }
 
-static ndt_t *
-ndt_copy_var_dim(const ndt_t *t, ndt_context_t *ctx)
+static const ndt_t *
+ndt_copy_var_dim(const ndt_t *t, bool opt, ndt_context_t *ctx)
 {
-    ndt_t *type;
     ndt_slice_t *slices;
     int nslices;
 
     assert(t->tag == VarDim);
 
-    type = ndt_copy(t->VarDim.type, ctx);
-    if (type == NULL) {
+    if (ndt_is_abstract(t))  {
+        return ndt_abstract_var_dim(t->VarDim.type, opt, ctx);
+    }
+
+    slices = NULL;
+    nslices = t->Concrete.VarDim.nslices;
+
+    if (nslices > 0) {
+        slices = ndt_alloc(nslices, sizeof *slices);
+        if (slices == NULL) {
+            return ndt_memory_error(ctx);
+        }
+        memcpy(slices, t->Concrete.VarDim.slices,
+               nslices * (sizeof *slices));
+    }
+
+    return ndt_var_dim(t->VarDim.type, t->Concrete.VarDim.offsets,
+                       nslices, slices,
+                       opt, ctx);
+}
+
+const ndt_t *
+ndt_convert_to_var_elem(const ndt_t *t, const ndt_t *type, int64_t index,
+                        ndt_context_t *ctx)
+{
+    ndt_t *u;
+    ndt_slice_t *slices;
+    int nslices;
+
+    if (t->tag != VarDim && t->tag != VarDimElem) {
+        ndt_err_format(ctx, NDT_ValueError,
+                       "ndt_convert_to_var_elem: need var dim as input");
         return NULL;
     }
 
     if (ndt_is_abstract(t))  {
-        return ndt_abstract_var_dim(type, ctx);
+        ndt_err_format(ctx, NDT_ValueError,
+                       "cannot convert abstract var dim into var elem");
+        return NULL;
+    }
+
+    if (ndt_is_optional(t))  {
+        ndt_err_format(ctx, NDT_ValueError,
+                       "cannot convert optional var dim into var elem");
+        return NULL;
     }
 
     slices = NULL;
@@ -78,14 +118,19 @@ ndt_copy_var_dim(const ndt_t *t, ndt_context_t *ctx)
                nslices * (sizeof *slices));
     }
 
-    return ndt_var_dim(type, ExternalOffsets,
-                       t->Concrete.VarDim.noffsets,
-                       t->Concrete.VarDim.offsets,
-                       nslices,
-                       slices, ctx);
+    u = (ndt_t *)ndt_var_dim(type, t->Concrete.VarDim.offsets,
+                             nslices, slices,
+                             false, ctx);
+    if (u == NULL) {
+        return NULL;
+    }
+
+    u->tag = VarDimElem;
+    u->VarDimElem.index = index;
+    return u;
 }
 
-static ndt_t *
+static const ndt_t *
 ndt_copy_function(const ndt_t *t, ndt_context_t *ctx)
 {
     ndt_t *u;
@@ -103,25 +148,22 @@ ndt_copy_function(const ndt_t *t, ndt_context_t *ctx)
     copy_common(u, t);
 
     for (i = 0; i < t->Function.nargs; i++) {
-        u->Function.types[i] = ndt_copy(t->Function.types[i], ctx);
-        if (u->Function.types[i] == NULL) {
-            ndt_del(u);
-            return NULL;
-        }
+        ndt_incref(t->Function.types[i]);
+        u->Function.types[i] = t->Function.types[i];
     }
 
     return u;
 }
 
-static ndt_t *
-ndt_copy_tuple(const ndt_t *t, ndt_context_t *ctx)
+static const ndt_t *
+ndt_copy_tuple(const ndt_t *t, bool opt, ndt_context_t *ctx)
 {
     ndt_t *u;
     int64_t i;
 
     assert(t->tag == Tuple);
 
-    u = ndt_tuple_new(t->Tuple.flag, t->Tuple.shape, ctx);
+    u = ndt_tuple_new(t->Tuple.flag, t->Tuple.shape, opt, ctx);
     if (u == NULL) {
         return NULL;
     }
@@ -129,11 +171,8 @@ ndt_copy_tuple(const ndt_t *t, ndt_context_t *ctx)
     copy_common(u, t);
 
     for (i = 0; i < t->Tuple.shape; i++) {
-        u->Tuple.types[i] = ndt_copy(t->Tuple.types[i], ctx);
-        if (u->Tuple.types[i] == NULL) {
-            ndt_del(u);
-            return NULL;
-        }
+        ndt_incref(t->Tuple.types[i]);
+        u->Tuple.types[i] = t->Tuple.types[i];
 
         u->Concrete.Tuple.offset[i] = t->Concrete.Tuple.offset[i];
         u->Concrete.Tuple.align[i] = t->Concrete.Tuple.align[i];
@@ -143,15 +182,15 @@ ndt_copy_tuple(const ndt_t *t, ndt_context_t *ctx)
     return u;
 }
 
-static ndt_t *
-ndt_copy_record(const ndt_t *t, ndt_context_t *ctx)
+static const ndt_t *
+ndt_copy_record(const ndt_t *t, bool opt, ndt_context_t *ctx)
 {
     ndt_t *u;
     int64_t i;
 
     assert(t->tag == Record);
 
-    u = ndt_record_new(t->Record.flag, t->Record.shape, ctx);
+    u = ndt_record_new(t->Record.flag, t->Record.shape, opt, ctx);
     if (u == NULL) {
         return NULL;
     }
@@ -161,15 +200,12 @@ ndt_copy_record(const ndt_t *t, ndt_context_t *ctx)
     for (i = 0; i < t->Record.shape; i++) {
         u->Record.names[i] = ndt_strdup(t->Record.names[i], ctx);
         if (u->Record.names[i] == NULL) {
-            ndt_del(u);
+            ndt_decref(u);
             return NULL;
         }
 
-        u->Record.types[i] = ndt_copy(t->Record.types[i], ctx);
-        if (u->Record.types[i] == NULL) {
-            ndt_del(u);
-            return NULL;
-        }
+        ndt_incref(t->Record.types[i]);
+        u->Record.types[i] = t->Record.types[i];
 
         u->Concrete.Record.offset[i] = t->Concrete.Record.offset[i];
         u->Concrete.Record.align[i] = t->Concrete.Record.align[i];
@@ -179,6 +215,35 @@ ndt_copy_record(const ndt_t *t, ndt_context_t *ctx)
     return u;
 }
 
+static const ndt_t *
+ndt_copy_union(const ndt_t *t, bool opt, ndt_context_t *ctx)
+{
+    ndt_t *u;
+    int64_t i;
+
+    assert(t->tag == Union);
+
+    u = ndt_new(t->Union.ntags, opt, ctx);
+    if (u == NULL) {
+        return NULL;
+    }
+
+    copy_common(u, t);
+
+    for (i = 0; i < t->Union.ntags; i++) {
+        u->Union.tags[i] = ndt_strdup(t->Union.tags[i], ctx);
+        if (u->Union.tags[i] == NULL) {
+            ndt_decref(u);
+            return NULL;
+        }
+
+        ndt_incref(t->Union.types[i]);
+        u->Union.types[i] = t->Union.types[i];
+    }
+
+    return u;
+}
+
 static int
 ndt_copy_value(ndt_value_t *v, const ndt_value_t *u, ndt_context_t *ctx)
 {
@@ -200,8 +265,8 @@ ndt_copy_value(ndt_value_t *v, const ndt_value_t *u, ndt_context_t *ctx)
     return -1;
 }
 
-static ndt_t *
-ndt_copy_categorical(const ndt_t *t, ndt_context_t *ctx)
+static const ndt_t *
+ndt_copy_categorical(const ndt_t *t, bool opt, ndt_context_t *ctx)
 {
     int64_t ntypes = t->Categorical.ntypes;
     ndt_value_t *types;
@@ -225,85 +290,79 @@ ndt_copy_categorical(const ndt_t *t, ndt_context_t *ctx)
         }
     }
 
-    return ndt_categorical(types, ntypes, ctx);
+    return ndt_categorical(types, ntypes, opt, ctx);
 }
 
-ndt_t *
+/* shallow copy */
+const ndt_t *
 ndt_copy(const ndt_t *t, ndt_context_t *ctx)
 {
+    bool opt = ndt_is_optional(t);
     ndt_t *u = NULL;
-    ndt_t *type;
 
     switch (t->tag) {
     case FixedDim: {
-        type = ndt_copy(t->FixedDim.type, ctx);
-        if (type == NULL) {
-            return NULL;
-        }
-
-        u = ndt_fixed_dim_tag(type, t->FixedDim.tag, t->FixedDim.shape,
-                              t->Concrete.FixedDim.step, ctx);
+        u = (ndt_t *)ndt_fixed_dim_tag(t->FixedDim.type, t->FixedDim.tag, t->FixedDim.shape,
+                                       t->Concrete.FixedDim.step, ctx);
         goto copy_common_fields;
     }
 
     case VarDim: {
-        u = ndt_copy_var_dim(t, ctx);
+        u = (ndt_t *)ndt_copy_var_dim(t, opt, ctx);
+        goto copy_common_fields;
+    }
+
+    case VarDimElem: {
+        u = (ndt_t *)ndt_copy_var_dim(t, opt, ctx);
+        u->VarDimElem.index = t->VarDimElem.index;
         goto copy_common_fields;
     }
 
     case SymbolicDim: {
         char *name;
 
-        type = ndt_copy(t->SymbolicDim.type, ctx);
-        if (type == NULL) {
-            return NULL;
-        }
-
         name = ndt_strdup(t->SymbolicDim.name, ctx);
         if (name == NULL) {
-            ndt_del(type);
             return NULL;
         }
 
-        u = ndt_symbolic_dim_tag(name, type, t->SymbolicDim.tag, ctx);
+        u = (ndt_t *)ndt_symbolic_dim_tag(name, t->SymbolicDim.type, t->SymbolicDim.tag, ctx);
         goto copy_common_fields;
     }
 
     case EllipsisDim: {
         char *name = NULL;
 
-        type = ndt_copy(t->EllipsisDim.type, ctx);
-        if (type == NULL) {
-            return NULL;
-        }
-
         if (t->EllipsisDim.name != NULL) {
             name = ndt_strdup(t->SymbolicDim.name, ctx);
             if (name == NULL) {
-                ndt_del(type);
                 return NULL;
             }
         }
 
-        u = ndt_ellipsis_dim_tag(name, type, t->EllipsisDim.tag, ctx);
+        u = (ndt_t *)ndt_ellipsis_dim_tag(name, t->EllipsisDim.type, t->EllipsisDim.tag, ctx);
+        goto copy_common_fields;
+    }
+
+    case Array: {
+        u = (ndt_t *)ndt_array(t->Array.type, opt, ctx);
         goto copy_common_fields;
     }
 
     case Tuple: {
-        return ndt_copy_tuple(t, ctx);
+        return ndt_copy_tuple(t, opt, ctx);
     }
 
     case Record: {
-        return ndt_copy_record(t, ctx);
+        return ndt_copy_record(t, opt, ctx);
     }
 
-    case Ref: {
-        type = ndt_copy(t->Ref.type, ctx);
-        if (type == NULL) {
-            return NULL;
-        }
+    case Union: {
+        return ndt_copy_union(t, opt, ctx);
+    }
 
-        u = ndt_ref(type, ctx);
+    case Ref: {
+        u = (ndt_t *)ndt_ref(t->Ref.type, opt, ctx);
         goto copy_common_fields;
     }
 
@@ -313,13 +372,7 @@ ndt_copy(const ndt_t *t, ndt_context_t *ctx)
             return NULL;
         }
 
-        type = ndt_copy(t->Constr.type, ctx);
-        if (type == NULL) {
-            ndt_free(name);
-            return NULL;
-        }
-
-        u = ndt_constr(name, type, ctx);
+        u = (ndt_t *)ndt_constr(name, t->Constr.type, opt, ctx);
         goto copy_common_fields;
     }
 
@@ -329,18 +382,12 @@ ndt_copy(const ndt_t *t, ndt_context_t *ctx)
             return NULL;
         }
 
-        type = ndt_copy(t->Nominal.type, ctx);
-        if (type == NULL) {
-            ndt_free(name);
-            return NULL;
-        }
-
-        u = ndt_nominal(name, type, ctx);
+        u = (ndt_t *)ndt_nominal(name, t->Nominal.type, opt, ctx);
         goto copy_common_fields;
     }
 
     case Categorical: {
-        u = ndt_copy_categorical(t, ctx);
+        u = (ndt_t *)ndt_copy_categorical(t, opt, ctx);
         goto copy_common_fields;
     }
 
@@ -352,7 +399,7 @@ ndt_copy(const ndt_t *t, ndt_context_t *ctx)
             return NULL;
         }
 
-        u = ndt_typevar(name, ctx);
+        u = (ndt_t *)ndt_typevar(name, ctx);
         goto copy_common_fields;
     }
 
@@ -363,18 +410,12 @@ ndt_copy(const ndt_t *t, ndt_context_t *ctx)
     case Module: {
         char *name;
 
-        type = ndt_copy(t->Module.type, ctx);
-        if (type == NULL) {
-            return NULL;
-        }
-
         name = ndt_strdup(t->Module.name, ctx);
         if (name == NULL) {
-            ndt_del(type);
             return NULL;
         }
 
-        u = ndt_module(name, type, ctx);
+        u = (ndt_t *)ndt_module(name, t->Module.type, ctx);
         goto copy_common_fields;
     }
 
@@ -383,21 +424,25 @@ ndt_copy(const ndt_t *t, ndt_context_t *ctx)
     case SignedKind: case UnsignedKind:
     case FloatKind: case ComplexKind:
     case FixedStringKind: case FixedBytesKind:
-    case Bool:
-    case Int8: case Int16: case Int32: case Int64:
-    case Uint8: case Uint16: case Uint32: case Uint64:
-    case Float16: case Float32: case Float64:
-    case Complex32: case Complex64: case Complex128:
     case FixedString: case FixedBytes:
-    case String: case Bytes:
+    case Bytes:
     case Char: {
-        u = ndt_new(t->tag, ctx);
+        u = ndt_new(t->tag, opt, ctx);
         if (u == NULL) {
             return NULL;
         }
         *u = *t;
+        u->refcnt = 1;
         return u;
-      }
+    }
+
+    case String:
+    case Bool:
+    case Int8: case Int16: case Int32: case Int64:
+    case Uint8: case Uint16: case Uint32: case Uint64:
+    case BFloat16: case Float16: case Float32: case Float64:
+    case BComplex32: case Complex32: case Complex64: case Complex128:
+        return t;
     }
 
     goto invalid_tag;
@@ -417,29 +462,34 @@ invalid_tag:
     return NULL;
 }
 
-static ndt_t *
-fixed_copy_contiguous(const ndt_t *t, ndt_t *type, ndt_context_t *ctx)
+static const ndt_t *
+fixed_copy_contiguous(const ndt_t *t, const ndt_t *type, ndt_context_t *ctx)
 {
+    const ndt_t *u, *v;
+
     if (t->ndim == 0) {
+        ndt_incref(type);
         return type;
     }
 
     assert(t->tag == FixedDim);
     assert(ndt_is_concrete(t));
-    type = fixed_copy_contiguous(t->FixedDim.type, type, ctx);
-    if (type == NULL) {
-        ndt_del(type);
+
+    u = fixed_copy_contiguous(t->FixedDim.type, type, ctx);
+    if (u == NULL) {
         return NULL;
     }
 
-    return ndt_fixed_dim_tag(type, t->FixedDim.tag, t->FixedDim.shape,
-                             INT64_MAX, ctx);
+    v = ndt_fixed_dim_tag(u, t->FixedDim.tag, t->FixedDim.shape,
+                          INT64_MAX, ctx);
+    ndt_decref(u);
+    return v;
 }
 
 typedef struct {
     int maxdim;
+    bool active[NDT_MAX_DIM+1];
     int32_t index[NDT_MAX_DIM+1];
-    int32_t noffsets[NDT_MAX_DIM+1];
     int32_t *offsets[NDT_MAX_DIM+1];
 } offsets_t;
 
@@ -452,60 +502,80 @@ clear_offsets(offsets_t *m)
 }
 
 static int
-var_init_offsets(offsets_t *m, const ndt_t *t, int32_t noffsets, ndt_context_t *ctx)
+var_init_offsets(offsets_t *m, ndt_context_t *ctx)
 {
     int32_t *offsets;
-    int64_t shape, start, step;
-    int64_t sum;
-    int32_t i;
-
-    assert(t->ndim >= 1);
 
-    offsets = ndt_alloc(noffsets, sizeof *offsets);
-    if (offsets == NULL) {
-        clear_offsets(m);
-        (void)ndt_memory_error(ctx);
-        return -1;
+    for (int i = 1; i <= m->maxdim; i++) {
+        offsets = ndt_calloc(m->index[i]+1, sizeof *offsets);
+        if (offsets == NULL) {
+            clear_offsets(m);
+            (void)ndt_memory_error(ctx);
+            return -1;
+        }
+        m->offsets[i] = offsets;
     }
-    m->noffsets[t->ndim] = noffsets;
-    m->offsets[t->ndim] = offsets;
 
-    if (t->ndim == 1) {
-        return 0;
+    return 0;
+}
+
+static int64_t
+get_index(int64_t shape, int64_t index, ndt_context_t *ctx)
+{
+    bool overflow = false;
+
+    if (index < 0) {
+        index = ADDi64(index, shape, &overflow);
     }
 
-    for (i=0, sum=0; i < noffsets-1; i++) {
-        shape = ndt_var_indices(&start, &step, t, i, ctx);
-        if (shape == -1) {
-            clear_offsets(m);
-            return -1;
-        }
-        sum += shape;
+    if (overflow || index < 0 || index >= shape) {
+        ndt_err_format(ctx, NDT_IndexError,
+            "index with value %" PRIi64 " out of bounds",
+            index);
+        return -1;
     }
 
-    return var_init_offsets(m, t->VarDim.type, (int32_t)sum+1, ctx);
+    return index;
 }
 
 static int
-var_copy_shapes(offsets_t *m, int64_t src_index, const ndt_t *t, ndt_context_t *ctx)
+var_copy_shapes(bool write, offsets_t *m, int64_t linear_index, const ndt_t *t,
+                ndt_context_t *ctx)
 {
     int64_t shape, start, step;
+    int64_t k;
 
     if (t->ndim == 0) {
         return 0;
     }
 
-    shape = ndt_var_indices(&start, &step, t, src_index, ctx);
+    shape = ndt_var_indices(&start, &step, t, linear_index, ctx);
     if (shape < 0) {
         clear_offsets(m);
         return -1;
     }
-    int32_t dst_index = m->index[t->ndim]++;
-    m->offsets[t->ndim][dst_index] = (int32_t)shape;
 
-    for (int64_t i = 0; i < shape; i++) {
-        int64_t src_next = start + i * step;
-        if (var_copy_shapes(m, src_next, t->VarDim.type, ctx) < 0) {
+    k = 0;
+    m->active[t->ndim] = true;
+
+    if (t->tag == VarDimElem) {
+        k = get_index(shape, t->VarDimElem.index, ctx);
+        if (k < 0) {
+            return -1;
+        }
+        shape = 1;
+        m->active[t->ndim] = false;
+    }
+
+    int32_t write_index = m->index[t->ndim]++;
+    if (write) {
+        int32_t sum = m->offsets[t->ndim][write_index];
+        m->offsets[t->ndim][write_index+1] = sum + (int32_t)shape;
+    }
+
+    for (int64_t i = k; i < k+shape; i++) {
+        int64_t next = start + i * step;
+        if (var_copy_shapes(write, m, next, t->VarDim.type, ctx) < 0) {
             clear_offsets(m);
             return -1;
         }
@@ -514,31 +584,34 @@ var_copy_shapes(offsets_t *m, int64_t src_index, const ndt_t *t, ndt_context_t *
     return 0;
 }
 
-static void
-var_sum_shapes(offsets_t *m)
+const ndt_t *
+var_from_offsets_and_dtype(offsets_t *m, const ndt_t *t, ndt_context_t *ctx)
 {
-    int i, k;
+    const ndt_t *u;
+    int i;
+
+    ndt_incref(t);
 
     for (i = 1; i <= m->maxdim; i++) {
-        int32_t sum = 0;
-        for (k = 0; k < m->noffsets[i]; k++) {
-            int32_t s = m->offsets[i][k];
-            m->offsets[i][k] = sum;
-            sum += s;
+        if (!m->active[i]) {
+            ndt_free(m->offsets[i]);
+            m->offsets[i] = NULL;
+            continue;
         }
-   }
-}
 
-ndt_t *
-var_from_offsets_and_dtype(offsets_t *m, ndt_t *type, ndt_context_t *ctx)
-{
-    ndt_t *t;
-    int i;
+        ndt_offsets_t *offsets = ndt_offsets_from_ptr(m->offsets[i], m->index[i]+1, ctx);
 
-    for (i=1, t=type; i <= m->maxdim; i++, type=t) {
-        t = ndt_var_dim(type, InternalOffsets, m->noffsets[i], m->offsets[i],
-                        0, NULL, ctx);
         m->offsets[i] = NULL;
+        if (offsets == NULL) {
+            ndt_decref(t);
+            clear_offsets(m);
+            return NULL;
+        }
+
+        u = ndt_var_dim(t, offsets, 0, NULL, false, ctx);
+        ndt_move(&t, u);
+        ndt_decref_offsets(offsets);
+
         if (t == NULL) {
             clear_offsets(m);
             return NULL;
@@ -548,33 +621,38 @@ var_from_offsets_and_dtype(offsets_t *m, ndt_t *type, ndt_context_t *ctx)
     return t;
 }
  
-static ndt_t *
-var_copy_contiguous(const ndt_t *t, ndt_t *dtype, ndt_context_t *ctx)
+static const ndt_t *
+var_copy_contiguous(const ndt_t *t, const ndt_t *dtype, int64_t linear_index,
+                    ndt_context_t *ctx)
 {
-    offsets_t m = {.maxdim=0, .index={0}, .noffsets={0}, .offsets={NULL}};
+    offsets_t m = {.maxdim=0, .index={0}, .offsets={NULL}};
 
-    assert(t->tag == VarDim);
     assert(ndt_is_concrete(t));
-    assert(t->Concrete.VarDim.noffsets == 2);
 
-    if (var_init_offsets(&m, t, 2, ctx) < 0) {
-        ndt_del(dtype);
+    m.maxdim = t->ndim;
+
+    if (var_copy_shapes(false, &m, linear_index, t, ctx) < 0) {
         return NULL;
     }
-    m.maxdim = t->ndim;
 
-    if (var_copy_shapes(&m, 0, t, ctx) < 0) {
-        ndt_del(dtype);
+    if (var_init_offsets(&m, ctx) < 0) {
         return NULL;
     }
 
-    var_sum_shapes(&m);
+    for (int i = 0; i <= m.maxdim; i++) {
+        m.index[i] = 0; 
+    }
+
+    if (var_copy_shapes(true, &m, linear_index, t, ctx) < 0) {
+        return NULL;
+    }
 
     return var_from_offsets_and_dtype(&m, dtype, ctx);
 }
 
-ndt_t *
-ndt_copy_contiguous_dtype(const ndt_t *t, ndt_t *dtype, ndt_context_t *ctx)
+const ndt_t *
+ndt_copy_contiguous_dtype(const ndt_t *t, const ndt_t *dtype, int64_t linear_index,
+                          ndt_context_t *ctx)
 {
     if (ndt_is_abstract(t) || ndt_is_abstract(dtype)) {
         ndt_err_format(ctx, NDT_ValueError,
@@ -586,29 +664,30 @@ ndt_copy_contiguous_dtype(const ndt_t *t, ndt_t *dtype, ndt_context_t *ctx)
     case FixedDim: {
         return fixed_copy_contiguous(t, dtype, ctx);
     }
-    case VarDim: {
-        return var_copy_contiguous(t, dtype, ctx);
+    case VarDim: case VarDimElem: {
+        return var_copy_contiguous(t, dtype, linear_index, ctx);
     }
     default:
+        ndt_incref(dtype);
         return dtype;
     }
 }
 
-ndt_t *
-ndt_copy_contiguous(const ndt_t *t, ndt_context_t *ctx)
+const ndt_t *
+ndt_copy_contiguous(const ndt_t *t, int64_t linear_index, ndt_context_t *ctx)
 {
-    ndt_t *dtype = ndt_copy(ndt_dtype(t), ctx);
-    if (dtype == NULL) {
-        return NULL;
-    }
+    const ndt_t *dtype = ndt_dtype(t);
 
-    return ndt_copy_contiguous_dtype(t, dtype, ctx);
+    return ndt_copy_contiguous_dtype(t, dtype, linear_index, ctx);
 }
 
-ndt_t *
-ndt_copy_abstract_var_dtype(const ndt_t *t, ndt_t *dtype, ndt_context_t *ctx)
+const ndt_t *
+ndt_copy_abstract_var_dtype(const ndt_t *t, const ndt_t *dtype, ndt_context_t *ctx)
 {
+    bool opt = ndt_is_optional(t);
+
     if (t->ndim == 0) {
+        ndt_incref(dtype);
         return dtype;
     }
 
@@ -619,12 +698,14 @@ ndt_copy_abstract_var_dtype(const ndt_t *t, ndt_t *dtype, ndt_context_t *ctx)
                 "ndt_copy_abstract_var_dtype() called on concrete type");
             return NULL;
         }
-        ndt_t *u = ndt_copy_abstract_var_dtype(t->VarDim.type, dtype, ctx);
+        const ndt_t *u = ndt_copy_abstract_var_dtype(t->VarDim.type, dtype, ctx);
         if (u == NULL) {
             return NULL;
         }
 
-        return ndt_abstract_var_dim(u, ctx);
+        const ndt_t *w = ndt_abstract_var_dim(u, opt, ctx);
+        ndt_decref(u);
+        return w;
     }
     default:
         ndt_err_format(ctx, NDT_ValueError,