llvm-runtime-resources-darwin 22.1.8 → 23.1.1

package/include/__clang_cuda_complex_builtins.h

@@ -16,19 +16,25 @@
 // to work with CUDA and OpenMP target offloading [in C and C++ mode].)
 
 #pragma push_macro("__DEVICE__")
-#if defined(__OPENMP_NVPTX__) || defined(__OPENMP_AMDGCN__)
+#if defined(__OPENMP_NVPTX__) || defined(__OPENMP_AMDGCN__) ||                 \
+    defined(__OPENMP_SPIRV__)
 #pragma omp declare target
 #define __DEVICE__ __attribute__((noinline, nothrow, cold, weak))
 #else
 #define __DEVICE__ __device__ inline
 #endif
 
-#ifdef __NVPTX__
+#if defined(__NVPTX__)
 // FIXME: NVPTX should use generic builtins.
 #define _SCALBNd __nv_scalbn
 #define _SCALBNf __nv_scalbnf
 #define _LOGBd __nv_logb
 #define _LOGBf __nv_logbf
+#elif defined(__OPENMP_SPIRV__)
+#define _SCALBNd __spirv_ocl_ldexp
+#define _SCALBNf __spirv_ocl_ldexp
+#define _LOGBd __spirv_ocl_logb
+#define _LOGBf __spirv_ocl_logb
 #else
 #define _SCALBNd __builtin_scalbn
 #define _SCALBNf __builtin_scalbnf
@@ -220,7 +226,8 @@ __DEVICE__ float _Complex __divsc3(float __a, float __b, float __c, float __d) {
 #undef _LOGBd
 #undef _LOGBf
 
-#if defined(__OPENMP_NVPTX__) || defined(__OPENMP_AMDGCN__)
+#if defined(__OPENMP_NVPTX__) || defined(__OPENMP_AMDGCN__) ||                 \
+    defined(__OPENMP_SPIRV__)
 #pragma omp end declare target
 #endif
 

package/include/__clang_cuda_intrinsics.h

@@ -449,33 +449,33 @@ inline __device__ double2 __ldg(const double2 *ptr) {
 inline __device__ unsigned __funnelshift_l(unsigned low32, unsigned high32,
                                            unsigned shiftWidth) {
   unsigned result;
-  asm("shf.l.wrap.b32 %0, %1, %2, %3;"
-      : "=r"(result)
-      : "r"(low32), "r"(high32), "r"(shiftWidth));
+  __asm__("shf.l.wrap.b32 %0, %1, %2, %3;"
+          : "=r"(result)
+          : "r"(low32), "r"(high32), "r"(shiftWidth));
   return result;
 }
 inline __device__ unsigned __funnelshift_lc(unsigned low32, unsigned high32,
                                             unsigned shiftWidth) {
   unsigned result;
-  asm("shf.l.clamp.b32 %0, %1, %2, %3;"
-      : "=r"(result)
-      : "r"(low32), "r"(high32), "r"(shiftWidth));
+  __asm__("shf.l.clamp.b32 %0, %1, %2, %3;"
+          : "=r"(result)
+          : "r"(low32), "r"(high32), "r"(shiftWidth));
   return result;
 }
 inline __device__ unsigned __funnelshift_r(unsigned low32, unsigned high32,
                                            unsigned shiftWidth) {
   unsigned result;
-  asm("shf.r.wrap.b32 %0, %1, %2, %3;"
-      : "=r"(result)
-      : "r"(low32), "r"(high32), "r"(shiftWidth));
+  __asm__("shf.r.wrap.b32 %0, %1, %2, %3;"
+          : "=r"(result)
+          : "r"(low32), "r"(high32), "r"(shiftWidth));
   return result;
 }
 inline __device__ unsigned __funnelshift_rc(unsigned low32, unsigned high32,
                                             unsigned shiftWidth) {
   unsigned ret;
-  asm("shf.r.clamp.b32 %0, %1, %2, %3;"
-      : "=r"(ret)
-      : "r"(low32), "r"(high32), "r"(shiftWidth));
+  __asm__("shf.r.clamp.b32 %0, %1, %2, %3;"
+          : "=r"(ret)
+          : "r"(low32), "r"(high32), "r"(shiftWidth));
   return ret;
 }
 
@@ -483,38 +483,40 @@ inline __device__ unsigned __funnelshift_rc(unsigned low32, unsigned high32,
 
 #pragma push_macro("__INTRINSIC_LOAD")
 #define __INTRINSIC_LOAD(__FnName, __AsmOp, __DeclType, __TmpType, __AsmType,  \
-                         __Clobber)                                            \
+                         __Volatile, __Clobber)                                \
   inline __device__ __DeclType __FnName(const __DeclType *__ptr) {             \
     __TmpType __ret;                                                           \
-    asm(__AsmOp " %0, [%1];" : __AsmType(__ret) : "l"(__ptr)__Clobber);        \
+    __asm__ __Volatile(__AsmOp " %0, [%1];"                                    \
+                       : __AsmType(__ret)                                      \
+                       : "l"(__ptr)__Clobber);                                 \
     return (__DeclType)__ret;                                                  \
   }
 
 #pragma push_macro("__INTRINSIC_LOAD2")
 #define __INTRINSIC_LOAD2(__FnName, __AsmOp, __DeclType, __TmpType, __AsmType, \
-                          __Clobber)                                           \
+                          __Volatile, __Clobber)                               \
   inline __device__ __DeclType __FnName(const __DeclType *__ptr) {             \
     __DeclType __ret;                                                          \
     __TmpType __tmp;                                                           \
-    asm(__AsmOp " {%0,%1}, [%2];"                                              \
-        : __AsmType(__tmp.x), __AsmType(__tmp.y)                               \
-        : "l"(__ptr)__Clobber);                                                \
+    __asm__ __Volatile(__AsmOp " {%0,%1}, [%2];"                               \
+                       : __AsmType(__tmp.x), __AsmType(__tmp.y)                \
+                       : "l"(__ptr)__Clobber);                                 \
     using __ElementType = decltype(__ret.x);                                   \
-    __ret.x = (__ElementType)(__tmp.x);                                        \
+    __ret.x = (__ElementType)__tmp.x;                                          \
     __ret.y = (__ElementType)__tmp.y;                                          \
     return __ret;                                                              \
   }
 
 #pragma push_macro("__INTRINSIC_LOAD4")
 #define __INTRINSIC_LOAD4(__FnName, __AsmOp, __DeclType, __TmpType, __AsmType, \
-                          __Clobber)                                           \
+                          __Volatile, __Clobber)                               \
   inline __device__ __DeclType __FnName(const __DeclType *__ptr) {             \
     __DeclType __ret;                                                          \
     __TmpType __tmp;                                                           \
-    asm(__AsmOp " {%0,%1,%2,%3}, [%4];"                                        \
-        : __AsmType(__tmp.x), __AsmType(__tmp.y), __AsmType(__tmp.z),          \
-          __AsmType(__tmp.w)                                                   \
-        : "l"(__ptr)__Clobber);                                                \
+    __asm__ __Volatile(__AsmOp " {%0,%1,%2,%3}, [%4];"                         \
+                       : __AsmType(__tmp.x), __AsmType(__tmp.y),               \
+                         __AsmType(__tmp.z), __AsmType(__tmp.w)                \
+                       : "l"(__ptr)__Clobber);                                 \
     using __ElementType = decltype(__ret.x);                                   \
     __ret.x = (__ElementType)__tmp.x;                                          \
     __ret.y = (__ElementType)__tmp.y;                                          \
@@ -523,179 +525,105 @@ inline __device__ unsigned __funnelshift_rc(unsigned low32, unsigned high32,
     return __ret;                                                              \
   }
 
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.s8", char, unsigned int, "=r", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.s8", signed char, unsigned int, "=r", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.s16", short, unsigned short, "=h", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.s32", int, unsigned int, "=r", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.s64", long long, unsigned long long,
-                 "=l", );
-
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.s8", char2, int2, "=r", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.s8", char4, int4, "=r", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.s16", short2, short2, "=h", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.s16", short4, short4, "=h", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.s32", int2, int2, "=r", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.s32", int4, int4, "=r", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.s64 ", longlong2, longlong2, "=l", );
-
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.u8", unsigned char, unsigned int,
-                 "=r", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.u16", unsigned short, unsigned short,
-                 "=h", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.u32", unsigned int, unsigned int,
-                 "=r", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.u64", unsigned long long,
-                 unsigned long long, "=l", );
-
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.u8", uchar2, int2, "=r", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.u8", uchar4, int4, "=r", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.u16", ushort2, ushort2, "=h", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.u16", ushort4, ushort4, "=h", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.u32", uint2, uint2, "=r", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.u32", uint4, uint4, "=r", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.u64", ulonglong2, ulonglong2,
-                  "=l", );
-
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.f32", float, float, "=f", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.f64", double, double, "=d", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.f32", float2, float2, "=f", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.f32", float4, float4, "=f", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.f64", double2, double2, "=d", );
-
-inline __device__ long __ldcg(const long *__ptr) {
-  unsigned long __ret;
-  if (sizeof(long) == 8) {
-    asm("ld.global.cg.s64 %0, [%1];" : "=l"(__ret) : "l"(__ptr));
-  } else {
-    asm("ld.global.cg.s32 %0, [%1];" : "=r"(__ret) : "l"(__ptr));
+#pragma push_macro("__INTRINSIC_LOAD_LONG")
+#define __INTRINSIC_LOAD_LONG(__Mode)                                          \
+  inline __device__ long __ld##__Mode(const long *__ptr) {                     \
+    if (__SIZEOF_LONG__ == __SIZEOF_LONG_LONG__) {                             \
+      return (long)__ld##__Mode((const long long *)__ptr);                     \
+    } else {                                                                   \
+      return (long)__ld##__Mode((const int *)__ptr);                           \
+    }                                                                          \
   }
-  return (long)__ret;
-}
-
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.u8", unsigned char, unsigned int,
-                 "=r", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.u16", unsigned short, unsigned short,
-                 "=h", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.u32", unsigned int, unsigned int,
-                 "=r", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.u64", unsigned long long,
-                 unsigned long long, "=l", : "memory");
-
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.s8", char, unsigned int,
-                 "=r", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.s8", signed char, unsigned int,
-                 "=r", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.s16", short, unsigned short,
-                 "=h", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.s32", int, unsigned int,
-                 "=r", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.s64", long long, unsigned long long,
-                 "=l", : "memory");
-
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.u8", uchar2, uint2,
-                  "=r", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.u8", uchar4, uint4,
-                  "=r", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.u16", ushort2, ushort2,
-                  "=h", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.u16", ushort4, ushort4,
-                  "=h", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.u32", uint2, uint2,
-                  "=r", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.u32", uint4, uint4,
-                  "=r", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.u64", ulonglong2, ulonglong2,
-                  "=l", : "memory");
-
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.s8", char2, int2, "=r", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.s8", char4, int4, "=r", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.s16", short2, short2,
-                  "=h", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.s16", short4, short4,
-                  "=h", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.s32", int2, int2, "=r", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.s32", int4, int4, "=r", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.s64", longlong2, longlong2,
-                  "=l", : "memory");
-
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.f32", float, float, "=f", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.f64", double, double, "=d", : "memory");
-
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.f32", float2, float2,
-                  "=f", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.f32", float4, float4,
-                  "=f", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.f64", double2, double2,
-                  "=d", : "memory");
-
-inline __device__ long __ldcv(const long *__ptr) {
-  unsigned long __ret;
-  if (sizeof(long) == 8) {
-    asm("ld.global.cv.s64 %0, [%1];" : "=l"(__ret) : "l"(__ptr));
-  } else {
-    asm("ld.global.cv.s32 %0, [%1];" : "=r"(__ret) : "l"(__ptr));
+
+#pragma push_macro("__INTRINSIC_LOAD_ULONG")
+#define __INTRINSIC_LOAD_ULONG(__Mode)                                         \
+  inline __device__ unsigned long __ld##__Mode(const unsigned long *__ptr) {   \
+    if (__SIZEOF_LONG__ == __SIZEOF_LONG_LONG__) {                             \
+      return (unsigned long)__ld##__Mode((const unsigned long long *)__ptr);   \
+    } else {                                                                   \
+      return (unsigned long)__ld##__Mode((const unsigned int *)__ptr);         \
+    }                                                                          \
   }
-  return (long)__ret;
-}
-
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.s8", char, unsigned int, "=r", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.s8", signed char, signed int, "=r", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.s16", short, unsigned short, "=h", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.s32", int, unsigned int, "=r", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.s64", long long, unsigned long long,
-                 "=l", );
-
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.s8", char2, int2, "=r", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.s8", char4, int4, "=r", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.s16", short2, short2, "=h", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.s16", short4, short4, "=h", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.s32", int2, int2, "=r", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.s32", int4, int4, "=r", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.s64", longlong2, longlong2, "=l", );
-
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.u8", unsigned char, unsigned int,
-                 "=r", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.u16", unsigned short, unsigned short,
-                 "=h", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.u32", unsigned int, unsigned int,
-                 "=r", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.u64", unsigned long long,
-                 unsigned long long, "=l", );
-
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.u8", uchar2, uint2, "=r", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.u8", uchar4, uint4, "=r", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.u16", ushort2, ushort2, "=h", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.u16", ushort4, ushort4, "=h", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.u32", uint2, uint2, "=r", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.u32", uint4, uint4, "=r", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.u64", ulonglong2, ulonglong2,
-                  "=l", );
-
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.f32", float, float, "=f", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.f64", double, double, "=d", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.f32", float2, float2, "=f", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.f32", float4, float4, "=f", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.f64", double2, double2, "=d", );
+
+#define __INTRINSIC_LOAD_FAMILY(__Mode, __Volatile, __Clobber)                 \
+  __INTRINSIC_LOAD(__ld##__Mode, "ld.global." #__Mode ".s8", char,             \
+                   unsigned int, "=r", __Volatile, __Clobber)                  \
+  __INTRINSIC_LOAD(__ld##__Mode, "ld.global." #__Mode ".s8", signed char,      \
+                   unsigned int, "=r", __Volatile, __Clobber)                  \
+  __INTRINSIC_LOAD(__ld##__Mode, "ld.global." #__Mode ".s16", short,           \
+                   unsigned short, "=h", __Volatile, __Clobber)                \
+  __INTRINSIC_LOAD(__ld##__Mode, "ld.global." #__Mode ".s32", int,             \
+                   unsigned int, "=r", __Volatile, __Clobber)                  \
+  __INTRINSIC_LOAD(__ld##__Mode, "ld.global." #__Mode ".s64", long long,       \
+                   unsigned long long, "=l", __Volatile, __Clobber)            \
+  __INTRINSIC_LOAD2(__ld##__Mode, "ld.global." #__Mode ".v2.s8", char2, int2,  \
+                    "=r", __Volatile, __Clobber)                               \
+  __INTRINSIC_LOAD4(__ld##__Mode, "ld.global." #__Mode ".v4.s8", char4, int4,  \
+                    "=r", __Volatile, __Clobber)                               \
+  __INTRINSIC_LOAD2(__ld##__Mode, "ld.global." #__Mode ".v2.s16", short2,      \
+                    short2, "=h", __Volatile, __Clobber)                       \
+  __INTRINSIC_LOAD4(__ld##__Mode, "ld.global." #__Mode ".v4.s16", short4,      \
+                    short4, "=h", __Volatile, __Clobber)                       \
+  __INTRINSIC_LOAD2(__ld##__Mode, "ld.global." #__Mode ".v2.s32", int2, int2,  \
+                    "=r", __Volatile, __Clobber)                               \
+  __INTRINSIC_LOAD4(__ld##__Mode, "ld.global." #__Mode ".v4.s32", int4, int4,  \
+                    "=r", __Volatile, __Clobber)                               \
+  __INTRINSIC_LOAD2(__ld##__Mode, "ld.global." #__Mode ".v2.s64", longlong2,   \
+                    longlong2, "=l", __Volatile, __Clobber)                    \
+  __INTRINSIC_LOAD(__ld##__Mode, "ld.global." #__Mode ".u8", unsigned char,    \
+                   unsigned int, "=r", __Volatile, __Clobber)                  \
+  __INTRINSIC_LOAD(__ld##__Mode, "ld.global." #__Mode ".u16", unsigned short,  \
+                   unsigned short, "=h", __Volatile, __Clobber)                \
+  __INTRINSIC_LOAD(__ld##__Mode, "ld.global." #__Mode ".u32", unsigned int,    \
+                   unsigned int, "=r", __Volatile, __Clobber)                  \
+  __INTRINSIC_LOAD(__ld##__Mode, "ld.global." #__Mode ".u64",                  \
+                   unsigned long long, unsigned long long, "=l", __Volatile,   \
+                   __Clobber)                                                  \
+  __INTRINSIC_LOAD2(__ld##__Mode, "ld.global." #__Mode ".v2.u8", uchar2,       \
+                    uint2, "=r", __Volatile, __Clobber)                        \
+  __INTRINSIC_LOAD4(__ld##__Mode, "ld.global." #__Mode ".v4.u8", uchar4,       \
+                    uint4, "=r", __Volatile, __Clobber)                        \
+  __INTRINSIC_LOAD2(__ld##__Mode, "ld.global." #__Mode ".v2.u16", ushort2,     \
+                    ushort2, "=h", __Volatile, __Clobber)                      \
+  __INTRINSIC_LOAD4(__ld##__Mode, "ld.global." #__Mode ".v4.u16", ushort4,     \
+                    ushort4, "=h", __Volatile, __Clobber)                      \
+  __INTRINSIC_LOAD2(__ld##__Mode, "ld.global." #__Mode ".v2.u32", uint2,       \
+                    uint2, "=r", __Volatile, __Clobber)                        \
+  __INTRINSIC_LOAD4(__ld##__Mode, "ld.global." #__Mode ".v4.u32", uint4,       \
+                    uint4, "=r", __Volatile, __Clobber)                        \
+  __INTRINSIC_LOAD2(__ld##__Mode, "ld.global." #__Mode ".v2.u64", ulonglong2,  \
+                    ulonglong2, "=l", __Volatile, __Clobber)                   \
+  __INTRINSIC_LOAD(__ld##__Mode, "ld.global." #__Mode ".f32", float, float,    \
+                   "=f", __Volatile, __Clobber)                                \
+  __INTRINSIC_LOAD(__ld##__Mode, "ld.global." #__Mode ".f64", double, double,  \
+                   "=d", __Volatile, __Clobber)                                \
+  __INTRINSIC_LOAD2(__ld##__Mode, "ld.global." #__Mode ".v2.f32", float2,      \
+                    float2, "=f", __Volatile, __Clobber)                       \
+  __INTRINSIC_LOAD4(__ld##__Mode, "ld.global." #__Mode ".v4.f32", float4,      \
+                    float4, "=f", __Volatile, __Clobber)                       \
+  __INTRINSIC_LOAD2(__ld##__Mode, "ld.global." #__Mode ".v2.f64", double2,     \
+                    double2, "=d", __Volatile, __Clobber)                      \
+  __INTRINSIC_LOAD_LONG(__Mode)                                                \
+  __INTRINSIC_LOAD_ULONG(__Mode)
+
+__INTRINSIC_LOAD_FAMILY(ca, __volatile__, /* no clobber */)
+__INTRINSIC_LOAD_FAMILY(cg, __volatile__, /* no clobber */)
+__INTRINSIC_LOAD_FAMILY(cs, __volatile__, /* no clobber */)
+__INTRINSIC_LOAD_FAMILY(cv, /* not volatile */, : "memory")
+__INTRINSIC_LOAD_FAMILY(lu, /* not volatile */, : "memory")
 
 #pragma pop_macro("__INTRINSIC_LOAD")
 #pragma pop_macro("__INTRINSIC_LOAD2")
 #pragma pop_macro("__INTRINSIC_LOAD4")
-
-inline __device__ long __ldcs(const long *__ptr) {
-  unsigned long __ret;
-  if (sizeof(long) == 8) {
-    asm("ld.global.cs.s64 %0, [%1];" : "=l"(__ret) : "l"(__ptr));
-  } else {
-    asm("ld.global.cs.s32 %0, [%1];" : "=r"(__ret) : "l"(__ptr));
-  }
-  return (long)__ret;
-}
+#pragma pop_macro("__INTRINSIC_LOAD_LONG")
+#pragma pop_macro("__INTRINSIC_LOAD_ULONG")
+#pragma pop_macro("__INTRINSIC_LOAD_FAMILY")
 
 #pragma push_macro("__INTRINSIC_STORE")
 #define __INTRINSIC_STORE(__FnName, __AsmOp, __DeclType, __TmpType, __AsmType) \
   inline __device__ void __FnName(__DeclType *__ptr, __DeclType __value) {     \
     __TmpType __tmp = (__TmpType)__value;                                      \
-    asm(__AsmOp " [%0], %1;" ::"l"(__ptr), __AsmType(__tmp) : "memory");       \
+    __asm__(__AsmOp " [%0], %1;" ::"l"(__ptr), __AsmType(__tmp) : "memory");   \
   }
 
 #pragma push_macro("__INTRINSIC_STORE2")
@@ -706,9 +634,9 @@ inline __device__ long __ldcs(const long *__ptr) {
     using __ElementType = decltype(__tmp.x);                                   \
     __tmp.x = (__ElementType)(__value.x);                                      \
     __tmp.y = (__ElementType)(__value.y);                                      \
-    asm(__AsmOp " [%0], {%1,%2};" ::"l"(__ptr), __AsmType(__tmp.x),            \
-        __AsmType(__tmp.y)                                                     \
-        : "memory");                                                           \
+    __asm__(__AsmOp " [%0], {%1,%2};" ::"l"(__ptr), __AsmType(__tmp.x),        \
+            __AsmType(__tmp.y)                                                 \
+            : "memory");                                                       \
   }
 
 #pragma push_macro("__INTRINSIC_STORE4")
@@ -721,49 +649,102 @@ inline __device__ long __ldcs(const long *__ptr) {
     __tmp.y = (__ElementType)(__value.y);                                      \
     __tmp.z = (__ElementType)(__value.z);                                      \
     __tmp.w = (__ElementType)(__value.w);                                      \
-    asm(__AsmOp " [%0], {%1,%2,%3,%4};" ::"l"(__ptr), __AsmType(__tmp.x),      \
-        __AsmType(__tmp.y), __AsmType(__tmp.z), __AsmType(__tmp.w)             \
-        : "memory");                                                           \
+    __asm__(__AsmOp " [%0], {%1,%2,%3,%4};" ::"l"(__ptr), __AsmType(__tmp.x),  \
+            __AsmType(__tmp.y), __AsmType(__tmp.z), __AsmType(__tmp.w)         \
+            : "memory");                                                       \
+  }
+
+#pragma push_macro("__INTRINSIC_STORE_LONG")
+#define __INTRINSIC_STORE_LONG(__Mode)                                         \
+  inline __device__ void __st##__Mode(long *__ptr, long __value) {             \
+    if (__SIZEOF_LONG__ == __SIZEOF_LONG_LONG__) {                             \
+      __st##__Mode((long long *)__ptr, (long long)__value);                    \
+    } else {                                                                   \
+      __st##__Mode((int *)__ptr, (int)__value);                                \
+    }                                                                          \
+  }
+
+#pragma push_macro("__INTRINSIC_STORE_ULONG")
+#define __INTRINSIC_STORE_ULONG(__Mode)                                        \
+  inline __device__ void __st##__Mode(unsigned long *__ptr,                    \
+                                      unsigned long __value) {                 \
+    if (__SIZEOF_LONG__ == __SIZEOF_LONG_LONG__) {                             \
+      __st##__Mode((unsigned long long *)__ptr, (unsigned long long)__value);  \
+    } else {                                                                   \
+      __st##__Mode((unsigned int *)__ptr, (unsigned int)__value);              \
+    }                                                                          \
   }
 
-__INTRINSIC_STORE(__stwt, "st.global.wt.s8", char, int, "r");
-__INTRINSIC_STORE(__stwt, "st.global.wt.s8", signed char, int, "r");
-__INTRINSIC_STORE(__stwt, "st.global.wt.s16", short, short, "h");
-__INTRINSIC_STORE(__stwt, "st.global.wt.s32", int, int, "r");
-__INTRINSIC_STORE(__stwt, "st.global.wt.s64", long long, long long, "l");
-
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.s8", char2, int2, "r");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.s8", char4, int4, "r");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.s16", short2, short2, "h");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.s16", short4, short4, "h");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.s32", int2, int2, "r");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.s32", int4, int4, "r");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.s64", longlong2, longlong2, "l");
-
-__INTRINSIC_STORE(__stwt, "st.global.wt.u8", unsigned char, int, "r");
-__INTRINSIC_STORE(__stwt, "st.global.wt.u16", unsigned short, unsigned short,
-                  "h");
-__INTRINSIC_STORE(__stwt, "st.global.wt.u32", unsigned int, unsigned int, "r");
-__INTRINSIC_STORE(__stwt, "st.global.wt.u64", unsigned long long,
-                  unsigned long long, "l");
-
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.u8", uchar2, uchar2, "r");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.u8", uchar4, uint4, "r");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.u16", ushort2, ushort2, "h");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.u16", ushort4, ushort4, "h");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.u32", uint2, uint2, "r");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.u32", uint4, uint4, "r");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.u64", ulonglong2, ulonglong2, "l");
-
-__INTRINSIC_STORE(__stwt, "st.global.wt.f32", float, float, "f");
-__INTRINSIC_STORE(__stwt, "st.global.wt.f64", double, double, "d");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.f32", float2, float2, "f");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.f32", float4, float4, "f");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.f64", double2, double2, "d");
+#pragma push_macro("__INTRINSIC_STORE_FAMILY")
+#define __INTRINSIC_STORE_FAMILY(__Mode)                                       \
+  __INTRINSIC_STORE(__st##__Mode, "st.global." #__Mode ".s8", char, int, "r")  \
+  __INTRINSIC_STORE(__st##__Mode, "st.global." #__Mode ".s8", signed char,     \
+                    int, "r")                                                  \
+  __INTRINSIC_STORE(__st##__Mode, "st.global." #__Mode ".s16", short, short,   \
+                    "h")                                                       \
+  __INTRINSIC_STORE(__st##__Mode, "st.global." #__Mode ".s32", int, int, "r")  \
+  __INTRINSIC_STORE(__st##__Mode, "st.global." #__Mode ".s64", long long,      \
+                    long long, "l")                                            \
+  __INTRINSIC_STORE2(__st##__Mode, "st.global." #__Mode ".v2.s8", char2, int2, \
+                     "r")                                                      \
+  __INTRINSIC_STORE4(__st##__Mode, "st.global." #__Mode ".v4.s8", char4, int4, \
+                     "r")                                                      \
+  __INTRINSIC_STORE2(__st##__Mode, "st.global." #__Mode ".v2.s16", short2,     \
+                     short2, "h")                                              \
+  __INTRINSIC_STORE4(__st##__Mode, "st.global." #__Mode ".v4.s16", short4,     \
+                     short4, "h")                                              \
+  __INTRINSIC_STORE2(__st##__Mode, "st.global." #__Mode ".v2.s32", int2, int2, \
+                     "r")                                                      \
+  __INTRINSIC_STORE4(__st##__Mode, "st.global." #__Mode ".v4.s32", int4, int4, \
+                     "r")                                                      \
+  __INTRINSIC_STORE2(__st##__Mode, "st.global." #__Mode ".v2.s64", longlong2,  \
+                     longlong2, "l")                                           \
+  __INTRINSIC_STORE(__st##__Mode, "st.global." #__Mode ".u8", unsigned char,   \
+                    int, "r")                                                  \
+  __INTRINSIC_STORE(__st##__Mode, "st.global." #__Mode ".u16", unsigned short, \
+                    unsigned short, "h")                                       \
+  __INTRINSIC_STORE(__st##__Mode, "st.global." #__Mode ".u32", unsigned int,   \
+                    unsigned int, "r")                                         \
+  __INTRINSIC_STORE(__st##__Mode, "st.global." #__Mode ".u64",                 \
+                    unsigned long long, unsigned long long, "l")               \
+  __INTRINSIC_STORE2(__st##__Mode, "st.global." #__Mode ".v2.u8", uchar2,      \
+                     uchar2, "r")                                              \
+  __INTRINSIC_STORE4(__st##__Mode, "st.global." #__Mode ".v4.u8", uchar4,      \
+                     uint4, "r")                                               \
+  __INTRINSIC_STORE2(__st##__Mode, "st.global." #__Mode ".v2.u16", ushort2,    \
+                     ushort2, "h")                                             \
+  __INTRINSIC_STORE4(__st##__Mode, "st.global." #__Mode ".v4.u16", ushort4,    \
+                     ushort4, "h")                                             \
+  __INTRINSIC_STORE2(__st##__Mode, "st.global." #__Mode ".v2.u32", uint2,      \
+                     uint2, "r")                                               \
+  __INTRINSIC_STORE4(__st##__Mode, "st.global." #__Mode ".v4.u32", uint4,      \
+                     uint4, "r")                                               \
+  __INTRINSIC_STORE2(__st##__Mode, "st.global." #__Mode ".v2.u64", ulonglong2, \
+                     ulonglong2, "l")                                          \
+  __INTRINSIC_STORE(__st##__Mode, "st.global." #__Mode ".f32", float, float,   \
+                    "f")                                                       \
+  __INTRINSIC_STORE(__st##__Mode, "st.global." #__Mode ".f64", double, double, \
+                    "d")                                                       \
+  __INTRINSIC_STORE2(__st##__Mode, "st.global." #__Mode ".v2.f32", float2,     \
+                     float2, "f")                                              \
+  __INTRINSIC_STORE4(__st##__Mode, "st.global." #__Mode ".v4.f32", float4,     \
+                     float4, "f")                                              \
+  __INTRINSIC_STORE2(__st##__Mode, "st.global." #__Mode ".v2.f64", double2,    \
+                     double2, "d")                                             \
+  __INTRINSIC_STORE_LONG(__Mode)                                               \
+  __INTRINSIC_STORE_ULONG(__Mode)
+
+__INTRINSIC_STORE_FAMILY(cg)
+__INTRINSIC_STORE_FAMILY(cs)
+__INTRINSIC_STORE_FAMILY(wb)
+__INTRINSIC_STORE_FAMILY(wt)
 
 #pragma pop_macro("__INTRINSIC_STORE")
 #pragma pop_macro("__INTRINSIC_STORE2")
 #pragma pop_macro("__INTRINSIC_STORE4")
+#pragma pop_macro("__INTRINSIC_STORE_LONG")
+#pragma pop_macro("__INTRINSIC_STORE_ULONG")
+#pragma pop_macro("__INTRINSIC_STORE_FAMILY")
 
 #endif // defined(__cplusplus) && (__cplusplus >= 201103L)
 #endif // !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 320

package/include/__clang_cuda_math_forward_declares.h

@@ -70,6 +70,11 @@ __DEVICE__ double floor(double);
 __DEVICE__ float floor(float);
 __DEVICE__ double fma(double, double, double);
 __DEVICE__ float fma(float, float, float);
+#ifdef _MSC_VER
+// long double fma variant is not actually supported by CUDA (PTX).
+// However, MS-STL requires that this is forward declared anyways.
+__DEVICE__ long double fma(long double, long double, long double);
+#endif
 __DEVICE__ double fmax(double, double);
 __DEVICE__ float fmax(float, float);
 __DEVICE__ double fmin(double, double);