xnd 0.2.0dev6 → 0.2.0dev7

data/ext/ruby_xnd/xnd/libxnd/contrib/bfloat16.h

@@ -0,0 +1,213 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+/* Modified and adapted for gumath. */
+
+#ifndef BFLOAT16_H
+#define BFLOAT16_H
+
+
+#include <stdint.h>
+#include <math.h>
+
+
+union FP32 {
+    unsigned int u;
+    float f;
+};
+
+// Converts a float point to bfloat16, with round-nearest-to-even as rounding
+// method.
+// TODO: There is a slightly faster implementation (8% faster on CPU)
+// than this (documented in cl/175987786), that is exponentially harder to
+// understand and document. Switch to the faster version when converting to
+// BF16 becomes compute-bound.
+static inline uint16_t
+xnd_round_to_bfloat16(float v)
+{
+    uint32_t input;
+    union FP32 f;
+    f.f = v;
+    input = f.u;
+    uint16_t output;
+
+    if (isnan(v)) {
+      // If the value is a NaN, squash it to a qNaN with msb of fraction set,
+      // this makes sure after truncation we don't end up with an inf.
+      //
+      // qNaN magic: All exponent bits set + most significant bit of fraction
+      // set.
+      output = 0x7fc0;
+    } else {
+      // Fast rounding algorithm that rounds a half value to nearest even. This
+      // reduces expected error when we convert a large number of floats. Here
+      // is how it works:
+      //
+      // Definitions:
+      // To convert a float 32 to bfloat16, a float 32 can be viewed as 32 bits
+      // with the following tags:
+      //
+      // Sign |  Exp (8 bits) | Frac (23 bits)
+      //  S     EEEEEEEE         FFFFFFLRTTTTTTTTTTTTTTT
+      //
+      //  S: Sign bit.
+      //  E: Exponent bits.
+      //  F: First 6 bits of fraction.
+      //  L: Least significant bit of resulting bfloat16 if we truncate away the
+      //  rest of the float32. This is also the 7th bit of fraction
+      //  R: Rounding bit, 8th bit of fraction.
+      //  T: Sticky bits, rest of fraction, 15 bits.
+      //
+      // To round half to nearest even, there are 3 cases where we want to round
+      // down (simply truncate the result of the bits away, which consists of
+      // rounding bit and sticky bits) and two cases where we want to round up
+      // (truncate then add one to the result).
+      //
+      // The fast converting algorithm simply adds lsb (L) to 0x7fff (15 bits of
+      // 1s) as the rounding bias, adds the rounding bias to the input, then
+      // truncates the last 16 bits away.
+      //
+      // To understand how it works, we can analyze this algorithm case by case:
+      //
+      // 1. L = 0, R = 0:
+      //   Expect: round down, this is less than half value.
+      //
+      //   Algorithm:
+      //   - Rounding bias: 0x7fff + 0 = 0x7fff
+      //   - Adding rounding bias to input may create any carry, depending on
+      //   whether there is any value set to 1 in T bits.
+      //   - R may be set to 1 if there is a carry.
+      //   - L remains 0.
+      //   - Note that this case also handles Inf and -Inf, where all fraction
+      //   bits, including L, R and Ts are all 0. The output remains Inf after
+      //   this algorithm.
+      //
+      // 2. L = 1, R = 0:
+      //   Expect: round down, this is less than half value.
+      //
+      //   Algorithm:
+      //   - Rounding bias: 0x7fff + 1 = 0x8000
+      //   - Adding rounding bias to input doesn't change sticky bits but
+      //   adds 1 to rounding bit.
+      //   - L remains 1.
+      //
+      // 3. L = 0, R = 1, all of T are 0:
+      //   Expect: round down, this is exactly at half, the result is already
+      //   even (L=0).
+      //
+      //   Algorithm:
+      //   - Rounding bias: 0x7fff + 0 = 0x7fff
+      //   - Adding rounding bias to input sets all sticky bits to 1, but
+      //   doesn't create a carry.
+      //   - R remains 1.
+      //   - L remains 0.
+      //
+      // 4. L = 1, R = 1:
+      //   Expect: round up, this is exactly at half, the result needs to be
+      //   round to the next even number.
+      //
+      //   Algorithm:
+      //   - Rounding bias: 0x7fff + 1 = 0x8000
+      //   - Adding rounding bias to input doesn't change sticky bits, but
+      //   creates a carry from rounding bit.
+      //   - The carry sets L to 0, creates another carry bit and propagate
+      //   forward to F bits.
+      //   - If all the F bits are 1, a carry then propagates to the exponent
+      //   bits, which then creates the minimum value with the next exponent
+      //   value. Note that we won't have the case where exponents are all 1,
+      //   since that's either a NaN (handled in the other if condition) or inf
+      //   (handled in case 1).
+      //
+      // 5. L = 0, R = 1, any of T is 1:
+      //   Expect: round up, this is greater than half.
+      //
+      //   Algorithm:
+      //   - Rounding bias: 0x7fff + 0 = 0x7fff
+      //   - Adding rounding bias to input creates a carry from sticky bits,
+      //   sets rounding bit to 0, then create another carry.
+      //   - The second carry sets L to 1.
+      //
+      // Examples:
+      //
+      //  Exact half value that is already even:
+      //    Input:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+      //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+      //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 1 0     1000000000000000
+      //
+      //     This falls into case 3. We truncate the rest of 16 bits and no
+      //     carry is created into F and L:
+      //
+      //    Output:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+      //     S     E E E E E E E E      F F F F F F L
+      //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 1 0
+      //
+      //  Exact half value, round to next even number:
+      //    Input:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+      //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+      //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 0 1     1000000000000000
+      //
+      //     This falls into case 4. We create a carry from R and T,
+      //     which then propagates into L and F:
+      //
+      //    Output:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+      //     S     E E E E E E E E      F F F F F F L
+      //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 1 0
+      //
+      //
+      //  Max denormal value round to min normal value:
+      //    Input:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+      //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+      //     0     0 0 0 0 0 0 0 0      1 1 1 1 1 1 1     1111111111111111
+      //
+      //     This falls into case 4. We create a carry from R and T,
+      //     propagate into L and F, which then propagates into exponent
+      //     bits:
+      //
+      //    Output:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+      //     S     E E E E E E E E      F F F F F F L
+      //     0     0 0 0 0 0 0 0 1      0 0 0 0 0 0 0
+      //
+      //  Max normal value round to Inf:
+      //    Input:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+      //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+      //     0     1 1 1 1 1 1 1 0      1 1 1 1 1 1 1     1111111111111111
+      //
+      //     This falls into case 4. We create a carry from R and T,
+      //     propagate into L and F, which then propagates into exponent
+      //     bits:
+      //
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+      //     S     E E E E E E E E      F F F F F F L
+      //     0     1 1 1 1 1 1 1 1      0 0 0 0 0 0 0
+      //
+      //
+      // Least significant bit of resulting bfloat.
+      uint32_t lsb = (input >> 16) & 1;
+      uint32_t rounding_bias = 0x7fff + lsb;
+      input += rounding_bias;
+      output = (uint16_t)(input >> 16);
+    }
+    return output;
+}
+
+
+#endif  // BFLOAT16_H

data/ext/ruby_xnd/xnd/libxnd/copy.c

@@ -38,6 +38,7 @@
 #include <assert.h>
 #include "ndtypes.h"
 #include "xnd.h"
+#include "overflow.h"
 #include "contrib.h"
 
 
@@ -164,6 +165,15 @@ copy_int64(xnd_t * const x, const int64_t i64, ndt_context_t *ctx)
         return 0;
     }
 
+    case BFloat16: {
+        if (i64 < -4503599627370496LL || i64 > 4503599627370496LL) {
+            return value_error(ctx);
+        }
+        double real = (double)i64;
+        xnd_bfloat_pack(x->ptr, real);
+        return 0;
+    }
+
     case Float16: {
         if (i64 < -4503599627370496LL || i64 > 4503599627370496LL) {
             return value_error(ctx);
@@ -189,6 +199,19 @@ copy_int64(xnd_t * const x, const int64_t i64, ndt_context_t *ctx)
         return 0;
     }
 
+    case BComplex32: {
+        if (i64 < -4503599627370496LL || i64 > 4503599627370496LL) {
+            return value_error(ctx);
+        }
+
+        double real = (double)i64;
+        double imag = 0.0;
+
+        xnd_bfloat_pack(x->ptr, real);
+        xnd_bfloat_pack(x->ptr+2, imag);
+        return 0;
+    }
+
     case Complex32: {
         if (i64 < -4503599627370496LL || i64 > 4503599627370496LL) {
             return value_error(ctx);
@@ -312,6 +335,15 @@ copy_uint64(xnd_t * const x, const uint64_t u64, ndt_context_t *ctx)
         return 0;
     }
 
+    case BFloat16: {
+        if (u64 > 4503599627370496LL) {
+            return value_error(ctx);
+        }
+        double real = (double)u64;
+        xnd_bfloat_pack(x->ptr, real);
+        return 0;
+    }
+
     case Float16: {
         if (u64 > 4503599627370496LL) {
             return value_error(ctx);
@@ -337,6 +369,19 @@ copy_uint64(xnd_t * const x, const uint64_t u64, ndt_context_t *ctx)
         return 0;
     }
 
+    case BComplex32: {
+        if (u64 > 4503599627370496LL) {
+            return value_error(ctx);
+        }
+
+        double real = (double)u64;
+        double imag = 0.0;
+
+        xnd_bfloat_pack(x->ptr, real);
+        xnd_bfloat_pack(x->ptr+2, imag);
+        return 0;
+    }
+
     case Complex32: {
         if (u64 > 4503599627370496LL) {
             return value_error(ctx);
@@ -474,6 +519,12 @@ copy_float64(xnd_t * const x, const double real, ndt_context_t *ctx)
         }
         uint64_t u64 = (uint64_t)real;
         PACK_SINGLE(x->ptr, u64, uint64_t, t->flags);
+        return 0;
+    }
+
+    case BFloat16: {
+        xnd_bfloat_pack(x->ptr, real);
+        return 0;
     }
 
     case Float16: {
@@ -489,6 +540,14 @@ copy_float64(xnd_t * const x, const double real, ndt_context_t *ctx)
         return 0;
     }
 
+    case BComplex32: {
+        double imag = 0.0;
+
+        xnd_bfloat_pack(x->ptr, real);
+        xnd_bfloat_pack(x->ptr+2, imag);
+        return 0;
+    }
+
     case Complex32: {
         double imag = 0.0;
 
@@ -532,7 +591,7 @@ copy_complex128(xnd_t * const x, const double real, const double imag,
     switch (t->tag) {
     case Int8: case Int16: case Int32: case Int64:
     case Uint8: case Uint16: case Uint32: case Uint64:
-    case Float16: case Float32: case Float64: {
+    case BFloat16: case Float16: case Float32: case Float64: {
        if (imag == 0.0) {
            return copy_float64(x, real, ctx);
        }
@@ -540,6 +599,12 @@ copy_complex128(xnd_t * const x, const double real, const double imag,
        return type_error(ctx);
     }
 
+    case BComplex32: {
+        xnd_bfloat_pack(x->ptr, real);
+        xnd_bfloat_pack(x->ptr+2, imag);
+        return 0;
+    }
+
     case Complex32: {
         if (xnd_float_pack2(real, (unsigned char *)x->ptr, le(t->flags), ctx) < 0) {
             return -1;
@@ -568,6 +633,8 @@ copy_complex128(xnd_t * const x, const double real, const double imag,
 int
 xnd_copy(xnd_t *y, const xnd_t *x, uint32_t flags, ndt_context_t *ctx)
 {
+    APPLY_STORED_INDICES_INT(x)
+    APPLY_STORED_INDICES_INT(y)
     const ndt_t * const t = x->type;
     const ndt_t * const u = y->type;
     int n;
@@ -583,6 +650,10 @@ xnd_copy(xnd_t *y, const xnd_t *x, uint32_t flags, ndt_context_t *ctx)
         return 0;
     }
 
+    if (ndt_is_optional(u)) {
+        xnd_set_valid(y);
+    }
+
     if (t->tag == Ref || u->tag == Ref) {
         return copy_ref(y, x, flags, ctx);
     }
@@ -688,6 +759,25 @@ xnd_copy(xnd_t *y, const xnd_t *x, uint32_t flags, ndt_context_t *ctx)
         return 0;
     }
 
+    case Union: {
+        if (!ndt_equal(u, t)) {
+            return type_error(ctx);
+        }
+
+        const xnd_t xnext = xnd_union_next(x, ctx);
+        if (xnext.ptr == NULL) {
+            return -1;
+        }
+
+        XND_UNION_TAG(y->ptr) = XND_UNION_TAG(x->ptr);
+        xnd_t ynext = xnd_union_next(y, ctx);
+        if (ynext.ptr == NULL) {
+            return -1;
+        }
+
+        return xnd_copy(&ynext, &xnext, flags, ctx);
+    }
+
     case Constr: {
         if (u->tag != Constr || strcmp(u->Constr.name, t->Constr.name) != 0) {
             return type_error(ctx);
@@ -805,6 +895,11 @@ xnd_copy(xnd_t *y, const xnd_t *x, uint32_t flags, ndt_context_t *ctx)
         return copy_uint64(y, u64, ctx);
     }
 
+    case BFloat16: {
+        double real = xnd_bfloat_unpack(x->ptr);
+        return copy_float64(y, real, ctx);
+    }
+
     case Float16: {
         double real = xnd_float_unpack2((unsigned char *)x->ptr, le(t->flags));
         return copy_float64(y, real, ctx);
@@ -820,6 +915,15 @@ xnd_copy(xnd_t *y, const xnd_t *x, uint32_t flags, ndt_context_t *ctx)
         return copy_float64(y, real, ctx);
     }
 
+    case BComplex32: {
+        double real, imag;
+
+        real = xnd_bfloat_unpack(x->ptr);
+        imag = xnd_bfloat_unpack(x->ptr+2);
+
+        return copy_complex128(y, real, imag, ctx);
+    }
+
     case Complex32: {
         double real, imag;
 
@@ -875,7 +979,7 @@ xnd_copy(xnd_t *y, const xnd_t *x, uint32_t flags, ndt_context_t *ctx)
             return type_error(ctx);
         }
 
-        s = ndt_strdup(XND_POINTER_DATA(x->ptr), ctx);
+        s = ndt_strdup(XND_STRING_DATA(x->ptr), ctx);
         if (s == NULL) {
             return -1;
         }
@@ -912,7 +1016,7 @@ xnd_copy(xnd_t *y, const xnd_t *x, uint32_t flags, ndt_context_t *ctx)
         memcpy(s, XND_BYTES_DATA(x->ptr), (size_t)size);
 
         if (XND_BYTES_DATA(y->ptr) != NULL) {
-            if (!(flags & XND_OWN_EMBEDDED)) {
+            if (!(flags & XND_OWN_BYTES)) {
                 ndt_err_format(ctx, NDT_RuntimeError,
                     "cannot free string pointer, xnd does not own it");
                 ndt_aligned_free(s);
@@ -926,8 +1030,55 @@ xnd_copy(xnd_t *y, const xnd_t *x, uint32_t flags, ndt_context_t *ctx)
         return 0;
     }
 
-    /* NOT REACHED: intercepted by equal_ref(). */
+    case Array: {
+        bool overflow = false;
+
+        if (u->tag != Array) {
+            return type_error(ctx);
+        }
+
+        const int64_t shape = XND_ARRAY_SHAPE(x->ptr);
+        const int64_t size = MULi64(shape, t->Array.itemsize, &overflow);
+        if (overflow) {
+            ndt_err_format(ctx, NDT_ValueError, "flexible array too large");
+            return -1;
+        }
+
+        char *data = ndt_aligned_calloc(u->align, size);
+        if (data == NULL) {
+            (void)ndt_memory_error(ctx);
+            return -1;
+        }
+
+        if (XND_ARRAY_DATA(y->ptr) != NULL) {
+            if (!(flags & XND_OWN_ARRAYS)) {
+                ndt_err_format(ctx, NDT_RuntimeError,
+                    "cannot free array data pointer, xnd does not own it");
+                ndt_aligned_free(data);
+                return -1;
+            }
+            ndt_aligned_free(XND_ARRAY_DATA(y->ptr));
+        }
+
+        XND_ARRAY_SHAPE(y->ptr) = shape;
+        XND_ARRAY_DATA(y->ptr) = data;
+
+        for (int64_t i = 0; i < shape; i++) {
+            const xnd_t xnext = xnd_array_next(x, i);
+            xnd_t ynext = xnd_array_next(y, i);
+            n = xnd_copy(&ynext, &xnext, flags, ctx);
+            if (n < 0) return n;
+        }
+
+        return 0;
+    }
+
+    /* NOT REACHED: intercepted by apply_stored_indices(). */
+    case VarDimElem:
+    /* NOT REACHED: intercepted by copy_ref(). */
     case Ref:
+        ndt_err_format(ctx, NDT_RuntimeError, "unexpected VarDimElem or Ref");
+        return -1;
 
     /* NOT REACHED: xnd types must be concrete. */
     case Module: case Function: