llvm-runtime-resources-darwin 0.0.0 → 23.1.0

package/include/amdgpuintrin.h

@@ -0,0 +1,164 @@
+//===-- amdgpuintrin.h - AMDPGU intrinsic functions -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __AMDGPUINTRIN_H
+#define __AMDGPUINTRIN_H
+
+#ifndef __AMDGPU__
+#error "This file is intended for AMDGPU targets or offloading to AMDGPU"
+#endif
+
+#ifndef __GPUINTRIN_H
+#error "Never use <amdgpuintrin.h> directly; include <gpuintrin.h> instead"
+#endif
+
+_Pragma("omp begin declare target device_type(nohost)");
+_Pragma("omp begin declare variant match(device = {arch(amdgcn)})");
+
+// Type aliases to the address spaces used by the AMDGPU backend.
+#define __gpu_private __attribute__((address_space(5)))
+#define __gpu_constant __attribute__((address_space(4)))
+#define __gpu_local __attribute__((address_space(3)))
+#define __gpu_global __attribute__((address_space(1)))
+#define __gpu_generic __attribute__((address_space(0)))
+
+// Returns the number of workgroups in the 'x' dimension of the grid.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_blocks_x(void) {
+  return __builtin_amdgcn_grid_size_x() / __builtin_amdgcn_workgroup_size_x();
+}
+
+// Returns the number of workgroups in the 'y' dimension of the grid.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_blocks_y(void) {
+  return __builtin_amdgcn_grid_size_y() / __builtin_amdgcn_workgroup_size_y();
+}
+
+// Returns the number of workgroups in the 'z' dimension of the grid.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_blocks_z(void) {
+  return __builtin_amdgcn_grid_size_z() / __builtin_amdgcn_workgroup_size_z();
+}
+
+// Returns the 'x' dimension of the current AMD workgroup's id.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_block_id_x(void) {
+  return __builtin_amdgcn_workgroup_id_x();
+}
+
+// Returns the 'y' dimension of the current AMD workgroup's id.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_block_id_y(void) {
+  return __builtin_amdgcn_workgroup_id_y();
+}
+
+// Returns the 'z' dimension of the current AMD workgroup's id.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_block_id_z(void) {
+  return __builtin_amdgcn_workgroup_id_z();
+}
+
+// Returns the number of workitems in the 'x' dimension.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_threads_x(void) {
+  return __builtin_amdgcn_workgroup_size_x();
+}
+
+// Returns the number of workitems in the 'y' dimension.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_threads_y(void) {
+  return __builtin_amdgcn_workgroup_size_y();
+}
+
+// Returns the number of workitems in the 'z' dimension.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_threads_z(void) {
+  return __builtin_amdgcn_workgroup_size_z();
+}
+
+// Returns the 'x' dimension id of the workitem in the current AMD workgroup.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_thread_id_x(void) {
+  return __builtin_amdgcn_workitem_id_x();
+}
+
+// Returns the 'y' dimension id of the workitem in the current AMD workgroup.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_thread_id_y(void) {
+  return __builtin_amdgcn_workitem_id_y();
+}
+
+// Returns the 'z' dimension id of the workitem in the current AMD workgroup.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_thread_id_z(void) {
+  return __builtin_amdgcn_workitem_id_z();
+}
+
+// Returns the size of an AMD wavefront, either 32 or 64 depending on hardware
+// and compilation options.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_lanes(void) {
+  return __builtin_amdgcn_wavefrontsize();
+}
+
+// Returns the id of the thread inside of an AMD wavefront executing together.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_lane_id(void) {
+  return __builtin_amdgcn_mbcnt_hi(~0u, __builtin_amdgcn_mbcnt_lo(~0u, 0u));
+}
+
+// Returns the bit-mask of active threads in the current wavefront.
+_DEFAULT_FN_ATTRS static __inline__ uint64_t __gpu_lane_mask(void) {
+  return __builtin_amdgcn_read_exec();
+}
+
+// Copies the value from the first active thread in the wavefront to the rest.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t
+__gpu_read_first_lane_u32(uint64_t __lane_mask, uint32_t __x) {
+  return __builtin_amdgcn_readfirstlane(__x);
+}
+
+// Returns a bitmask of threads in the current lane for which \p x is true.
+_DEFAULT_FN_ATTRS static __inline__ uint64_t __gpu_ballot(uint64_t __lane_mask,
+                                                          bool __x) {
+  // The lane_mask & gives the nvptx semantics when lane_mask is a subset of
+  // the active threads
+  return __lane_mask & __builtin_amdgcn_ballot_w64(__x);
+}
+
+// Waits for all the threads in the block to converge and issues a fence.
+_DEFAULT_FN_ATTRS static __inline__ void __gpu_sync_threads(void) {
+  __builtin_amdgcn_s_barrier();
+  __builtin_amdgcn_fence(__ATOMIC_SEQ_CST, "workgroup");
+}
+
+// Wait for all threads in the wavefront to converge, this is a noop on AMDGPU.
+_DEFAULT_FN_ATTRS static __inline__ void __gpu_sync_lane(uint64_t __lane_mask) {
+  __builtin_amdgcn_wave_barrier();
+}
+
+// Shuffles the the lanes inside the wavefront according to the given index.
+_DEFAULT_FN_ATTRS static __inline__ uint32_t
+__gpu_shuffle_idx_u32(uint64_t __lane_mask, uint32_t __idx, uint32_t __x,
+                      uint32_t __width) {
+  uint32_t __lane = __idx + (__gpu_lane_id() & ~(__width - 1));
+  return __builtin_amdgcn_ds_bpermute(__lane << 2, __x);
+}
+
+// Returns true if the flat pointer points to AMDGPU 'shared' memory.
+_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_local(const void *ptr) {
+  return __builtin_amdgcn_is_shared((void [[clang::address_space(0)]] *)((
+      void [[clang::opencl_generic]] *)ptr));
+}
+
+// Returns true if the flat pointer points to AMDGPU 'private' memory.
+_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_private(const void *ptr) {
+  return __builtin_amdgcn_is_private((void [[clang::address_space(0)]] *)((
+      void [[clang::opencl_generic]] *)ptr));
+}
+
+// Terminates execution of the associated wavefront.
+_DEFAULT_FN_ATTRS [[noreturn]] static __inline__ void __gpu_exit(void) {
+  __builtin_amdgcn_endpgm();
+}
+
+// Suspend the thread briefly to assist the scheduler during busy loops.
+_DEFAULT_FN_ATTRS static __inline__ void __gpu_thread_suspend(void) {
+  __builtin_amdgcn_s_sleep(2);
+}
+
+_Pragma("omp end declare variant");
+_Pragma("omp end declare target");
+
+#endif // __AMDGPUINTRIN_H

package/include/amdhsa_abi.h

@@ -0,0 +1,83 @@
+//===-- amdhsa_abi.h - AMDHSA ABI definition utilities --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __AMDHSA_ABI_H
+#define __AMDHSA_ABI_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct __attribute__((aligned(8))) amdhsa_implicit_kernarg_v5 {
+  uint32_t block_count[3];
+  uint16_t group_size[3];
+  uint16_t remainder[3];
+  char reserved0[16];
+  uint64_t global_offset[3];
+  uint16_t grid_dims;
+  char reserved1[6];
+  __attribute__((opencl_global)) void *printf_buffer;
+  __attribute__((opencl_global)) void *hostcall_buffer;
+  __attribute__((opencl_global)) void *multigrid_sync_arg;
+  __attribute__((opencl_global)) void *heap_v1;
+  __attribute__((opencl_global)) void *default_queue;
+  __attribute__((opencl_global)) void *completion_action;
+  char reserved2[72];
+  uint32_t private_base; // Unused on gfx9+
+  uint32_t shared_base;  // Unused on gfx9+
+  __attribute__((opencl_global)) void *queue_ptr;
+  char reserved3[48];
+} amdhsa_implicit_kernarg_v5;
+
+_Static_assert(sizeof(amdhsa_implicit_kernarg_v5) == 256, "wrong struct size");
+
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, block_count[0]) == 0,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, block_count[1]) == 4,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, block_count[2]) == 8,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, group_size[0]) == 12,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, group_size[1]) == 14,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, group_size[2]) == 16,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, remainder[0]) == 18,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, remainder[1]) == 20,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, remainder[2]) == 22,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, global_offset[0]) == 40,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, global_offset[1]) == 48,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, global_offset[2]) == 56,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, grid_dims) == 64,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, printf_buffer) == 72,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, hostcall_buffer) == 80,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, multigrid_sync_arg) == 88,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, heap_v1) == 96,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, default_queue) == 104,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, completion_action) == 112,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, private_base) == 192,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, shared_base) == 196,
+               "wrong offset");
+_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, queue_ptr) == 200,
+               "wrong offset");
+
+#endif // __AMDHSA_ABI_H

package/include/ammintrin.h

@@ -0,0 +1,183 @@
+/*===---- ammintrin.h - SSE4a intrinsics -----------------------------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __AMMINTRIN_H
+#define __AMMINTRIN_H
+
+#if !defined(__i386__) && !defined(__x86_64__)
+#error "This header is only meant to be used on x86 and x64 architecture"
+#endif
+
+#include <pmmintrin.h>
+
+/* Define the default attributes for the functions in this file. */
+#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sse4a"), __min_vector_width__(128)))
+
+/// Extracts the specified bits from the lower 64 bits of the 128-bit
+///    integer vector operand at the index \a idx and of the length \a len.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \code
+/// __m128i _mm_extracti_si64(__m128i x, const int len, const int idx);
+/// \endcode
+///
+/// This intrinsic corresponds to the <c> EXTRQ </c> instruction.
+///
+/// \param x
+///    The value from which bits are extracted.
+/// \param len
+///    Bits [5:0] specify the length; the other bits are ignored. If bits [5:0]
+///    are zero, the length is interpreted as 64.
+/// \param idx
+///    Bits [5:0] specify the index of the least significant bit; the other
+///    bits are ignored. If the sum of the index and length is greater than 64,
+///    the result is undefined. If the length and index are both zero, bits
+///    [63:0] of parameter \a x are extracted. If the length is zero but the
+///    index is non-zero, the result is undefined.
+/// \returns A 128-bit integer vector whose lower 64 bits contain the bits
+///    extracted from the source operand.
+#define _mm_extracti_si64(x, len, idx) \
+  ((__m128i)__builtin_ia32_extrqi((__v2di)(__m128i)(x), \
+                                  (char)(len), (char)(idx)))
+
+/// Extracts the specified bits from the lower 64 bits of the 128-bit
+///    integer vector operand at the index and of the length specified by
+///    \a __y.
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> EXTRQ </c> instruction.
+///
+/// \param __x
+///    The value from which bits are extracted.
+/// \param __y
+///    Specifies the index of the least significant bit at [13:8] and the
+///    length at [5:0]; all other bits are ignored. If bits [5:0] are zero, the
+///    length is interpreted as 64. If the sum of the index and length is
+///    greater than 64, the result is undefined. If the length and index are
+///    both zero, bits [63:0] of parameter \a __x are extracted. If the length
+///    is zero but the index is non-zero, the result is undefined.
+/// \returns A 128-bit vector whose lower 64 bits contain the bits extracted
+///    from the source operand.
+static __inline__ __m128i __DEFAULT_FN_ATTRS
+_mm_extract_si64(__m128i __x, __m128i __y)
+{
+  return (__m128i)__builtin_ia32_extrq((__v2di)__x, (__v16qi)__y);
+}
+
+/// Inserts bits of a specified length from the source integer vector
+///    \a y into the lower 64 bits of the destination integer vector \a x at
+///    the index \a idx and of the length \a len.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \code
+/// __m128i _mm_inserti_si64(__m128i x, __m128i y, const int len,
+/// const int idx);
+/// \endcode
+///
+/// This intrinsic corresponds to the <c> INSERTQ </c> instruction.
+///
+/// \param x
+///    The destination operand where bits will be inserted. The inserted bits
+///    are defined by the length \a len and by the index \a idx specifying the
+///    least significant bit.
+/// \param y
+///    The source operand containing the bits to be extracted. The extracted
+///    bits are the least significant bits of operand \a y of length \a len.
+/// \param len
+///    Bits [5:0] specify the length; the other bits are ignored. If bits [5:0]
+///    are zero, the length is interpreted as 64.
+/// \param idx
+///    Bits [5:0] specify the index of the least significant bit; the other
+///    bits are ignored. If the sum of the index and length is greater than 64,
+///    the result is undefined. If the length and index are both zero, bits
+///    [63:0] of parameter \a y are inserted into parameter \a x. If the length
+///    is zero but the index is non-zero, the result is undefined.
+/// \returns A 128-bit integer vector containing the original lower 64-bits of
+///    destination operand \a x with the specified bitfields replaced by the
+///    lower bits of source operand \a y. The upper 64 bits of the return value
+///    are undefined.
+#define _mm_inserti_si64(x, y, len, idx) \
+  ((__m128i)__builtin_ia32_insertqi((__v2di)(__m128i)(x), \
+                                    (__v2di)(__m128i)(y), \
+                                    (char)(len), (char)(idx)))
+
+/// Inserts bits of a specified length from the source integer vector
+///    \a __y into the lower 64 bits of the destination integer vector \a __x
+///    at the index and of the length specified by \a __y.
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> INSERTQ </c> instruction.
+///
+/// \param __x
+///    The destination operand where bits will be inserted. The inserted bits
+///    are defined by the length and by the index of the least significant bit
+///    specified by operand \a __y.
+/// \param __y
+///    The source operand containing the bits to be extracted. The extracted
+///    bits are the least significant bits of operand \a __y with length
+///    specified by bits [69:64]. These are inserted into the destination at the
+///    index specified by bits [77:72]; all other bits are ignored. If bits
+///    [69:64] are zero, the length is interpreted as 64. If the sum of the
+///    index and length is greater than 64, the result is undefined. If the
+///    length and index are both zero, bits [63:0] of parameter \a __y are
+///    inserted into parameter \a __x. If the length is zero but the index is
+///    non-zero, the result is undefined.
+/// \returns A 128-bit integer vector containing the original lower 64-bits of
+///    destination operand \a __x with the specified bitfields replaced by the
+///    lower bits of source operand \a __y. The upper 64 bits of the return
+///    value are undefined.
+static __inline__ __m128i __DEFAULT_FN_ATTRS
+_mm_insert_si64(__m128i __x, __m128i __y)
+{
+  return (__m128i)__builtin_ia32_insertq((__v2di)__x, (__v2di)__y);
+}
+
+/// Stores a 64-bit double-precision value in a 64-bit memory location.
+///    To minimize caching, the data is flagged as non-temporal (unlikely to be
+///    used again soon).
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> MOVNTSD </c> instruction.
+///
+/// \param __p
+///    The 64-bit memory location used to store the register value.
+/// \param __a
+///    The 64-bit double-precision floating-point register value to be stored.
+static __inline__ void __DEFAULT_FN_ATTRS
+_mm_stream_sd(void *__p, __m128d __a)
+{
+  __builtin_ia32_movntsd((double *)__p, (__v2df)__a);
+}
+
+/// Stores a 32-bit single-precision floating-point value in a 32-bit
+///    memory location. To minimize caching, the data is flagged as
+///    non-temporal (unlikely to be used again soon).
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> MOVNTSS </c> instruction.
+///
+/// \param __p
+///    The 32-bit memory location used to store the register value.
+/// \param __a
+///    The 32-bit single-precision floating-point register value to be stored.
+static __inline__ void __DEFAULT_FN_ATTRS
+_mm_stream_ss(void *__p, __m128 __a)
+{
+  __builtin_ia32_movntss((float *)__p, (__v4sf)__a);
+}
+
+#undef __DEFAULT_FN_ATTRS
+
+#endif /* __AMMINTRIN_H */

package/include/amo.h

@@ -0,0 +1,220 @@
+/*===---- amo.h - PowerPC Atomic Memory Operations ------------------------===*\
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+\*===----------------------------------------------------------------------===*/
+
+/* This header provides compatibility for GCC's AMO functions.
+ * The functions here call Clang's underlying AMO builtins.
+ */
+
+#ifndef _AMO_H
+#define _AMO_H
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* AMO Load Operation Codes (FC values) */
+enum {
+  _AMO_LD_ADD = 0x00,  /* Fetch and Add */
+  _AMO_LD_XOR = 0x01,  /* Fetch and XOR */
+  _AMO_LD_IOR = 0x02,  /* Fetch and OR */
+  _AMO_LD_AND = 0x03,  /* Fetch and AND */
+  _AMO_LD_UMAX = 0x04, /* Fetch and Maximum Unsigned */
+  _AMO_LD_SMAX = 0x05, /* Fetch and Maximum Signed */
+  _AMO_LD_UMIN = 0x06, /* Fetch and Minimum Unsigned */
+  _AMO_LD_SMIN = 0x07, /* Fetch and Minimum Signed */
+  _AMO_LD_SWAP = 0x08  /* Swap */
+};
+
+/* 32-bit unsigned AMO load operations */
+static inline uint32_t amo_lwat_add(uint32_t *ptr, uint32_t val) {
+  return __builtin_amo_lwat(ptr, val, _AMO_LD_ADD);
+}
+
+static inline uint32_t amo_lwat_xor(uint32_t *ptr, uint32_t val) {
+  return __builtin_amo_lwat(ptr, val, _AMO_LD_XOR);
+}
+
+static inline uint32_t amo_lwat_ior(uint32_t *ptr, uint32_t val) {
+  return __builtin_amo_lwat(ptr, val, _AMO_LD_IOR);
+}
+
+static inline uint32_t amo_lwat_and(uint32_t *ptr, uint32_t val) {
+  return __builtin_amo_lwat(ptr, val, _AMO_LD_AND);
+}
+
+static inline uint32_t amo_lwat_umax(uint32_t *ptr, uint32_t val) {
+  return __builtin_amo_lwat(ptr, val, _AMO_LD_UMAX);
+}
+
+static inline uint32_t amo_lwat_umin(uint32_t *ptr, uint32_t val) {
+  return __builtin_amo_lwat(ptr, val, _AMO_LD_UMIN);
+}
+
+static inline uint32_t amo_lwat_swap(uint32_t *ptr, uint32_t val) {
+  return __builtin_amo_lwat(ptr, val, _AMO_LD_SWAP);
+}
+
+/* 32-bit signed AMO load operations */
+static inline int32_t amo_lwat_sadd(int32_t *ptr, int32_t val) {
+  return __builtin_amo_lwat_s(ptr, val, _AMO_LD_ADD);
+}
+
+static inline int32_t amo_lwat_smax(int32_t *ptr, int32_t val) {
+  return __builtin_amo_lwat_s(ptr, val, _AMO_LD_SMAX);
+}
+
+static inline int32_t amo_lwat_smin(int32_t *ptr, int32_t val) {
+  return __builtin_amo_lwat_s(ptr, val, _AMO_LD_SMIN);
+}
+
+static inline int32_t amo_lwat_sswap(int32_t *ptr, int32_t val) {
+  return __builtin_amo_lwat_s(ptr, val, _AMO_LD_SWAP);
+}
+
+/* 64-bit unsigned AMO load operations */
+static inline uint64_t amo_ldat_add(uint64_t *ptr, uint64_t val) {
+  return __builtin_amo_ldat(ptr, val, _AMO_LD_ADD);
+}
+
+static inline uint64_t amo_ldat_xor(uint64_t *ptr, uint64_t val) {
+  return __builtin_amo_ldat(ptr, val, _AMO_LD_XOR);
+}
+
+static inline uint64_t amo_ldat_ior(uint64_t *ptr, uint64_t val) {
+  return __builtin_amo_ldat(ptr, val, _AMO_LD_IOR);
+}
+
+static inline uint64_t amo_ldat_and(uint64_t *ptr, uint64_t val) {
+  return __builtin_amo_ldat(ptr, val, _AMO_LD_AND);
+}
+
+static inline uint64_t amo_ldat_umax(uint64_t *ptr, uint64_t val) {
+  return __builtin_amo_ldat(ptr, val, _AMO_LD_UMAX);
+}
+
+static inline uint64_t amo_ldat_umin(uint64_t *ptr, uint64_t val) {
+  return __builtin_amo_ldat(ptr, val, _AMO_LD_UMIN);
+}
+
+static inline uint64_t amo_ldat_swap(uint64_t *ptr, uint64_t val) {
+  return __builtin_amo_ldat(ptr, val, _AMO_LD_SWAP);
+}
+
+/* 64-bit signed AMO load operations */
+static inline int64_t amo_ldat_sadd(int64_t *ptr, int64_t val) {
+  return __builtin_amo_ldat_s(ptr, val, _AMO_LD_ADD);
+}
+
+static inline int64_t amo_ldat_smax(int64_t *ptr, int64_t val) {
+  return __builtin_amo_ldat_s(ptr, val, _AMO_LD_SMAX);
+}
+
+static inline int64_t amo_ldat_smin(int64_t *ptr, int64_t val) {
+  return __builtin_amo_ldat_s(ptr, val, _AMO_LD_SMIN);
+}
+
+static inline int64_t amo_ldat_sswap(int64_t *ptr, int64_t val) {
+  return __builtin_amo_ldat_s(ptr, val, _AMO_LD_SWAP);
+}
+
+/* AMO Store Operation Codes (FC values) */
+enum _AMO_ST {
+  _AMO_ST_ADD = 0x00,  /* Store Add */
+  _AMO_ST_XOR = 0x01,  /* Store Xor */
+  _AMO_ST_IOR = 0x02,  /* Store Ior */
+  _AMO_ST_AND = 0x03,  /* Store And */
+  _AMO_ST_UMAX = 0x04, /* Store Unsigned Maximum */
+  _AMO_ST_SMAX = 0x05, /* Store Signed Maximum */
+  _AMO_ST_UMIN = 0x06, /* Store Unsigned Minimum */
+  _AMO_ST_SMIN = 0x07, /* Store Signed Minimum */
+  _AMO_ST_TWIN = 0x18  /* Store Twin */
+};
+
+/* 32-bit unsigned AMO store operations */
+static inline void amo_stwat_add(uint32_t *ptr, uint32_t val) {
+  __builtin_amo_stwat(ptr, val, _AMO_ST_ADD);
+}
+
+static inline void amo_stwat_xor(uint32_t *ptr, uint32_t val) {
+  __builtin_amo_stwat(ptr, val, _AMO_ST_XOR);
+}
+
+static inline void amo_stwat_ior(uint32_t *ptr, uint32_t val) {
+  __builtin_amo_stwat(ptr, val, _AMO_ST_IOR);
+}
+
+static inline void amo_stwat_and(uint32_t *ptr, uint32_t val) {
+  __builtin_amo_stwat(ptr, val, _AMO_ST_AND);
+}
+
+static inline void amo_stwat_umax(uint32_t *ptr, uint32_t val) {
+  __builtin_amo_stwat(ptr, val, _AMO_ST_UMAX);
+}
+
+static inline void amo_stwat_umin(uint32_t *ptr, uint32_t val) {
+  __builtin_amo_stwat(ptr, val, _AMO_ST_UMIN);
+}
+
+/* 32-bit signed AMO store operations */
+static inline void amo_stwat_sadd(int32_t *ptr, int32_t val) {
+  __builtin_amo_stwat_s(ptr, val, _AMO_ST_ADD);
+}
+
+static inline void amo_stwat_smax(int32_t *ptr, int32_t val) {
+  __builtin_amo_stwat_s(ptr, val, _AMO_ST_SMAX);
+}
+
+static inline void amo_stwat_smin(int32_t *ptr, int32_t val) {
+  __builtin_amo_stwat_s(ptr, val, _AMO_ST_SMIN);
+}
+
+/* 64-bit unsigned AMO store operations */
+static inline void amo_stdat_add(uint64_t *ptr, uint64_t val) {
+  __builtin_amo_stdat(ptr, val, _AMO_ST_ADD);
+}
+
+static inline void amo_stdat_xor(uint64_t *ptr, uint64_t val) {
+  __builtin_amo_stdat(ptr, val, _AMO_ST_XOR);
+}
+
+static inline void amo_stdat_ior(uint64_t *ptr, uint64_t val) {
+  __builtin_amo_stdat(ptr, val, _AMO_ST_IOR);
+}
+
+static inline void amo_stdat_and(uint64_t *ptr, uint64_t val) {
+  __builtin_amo_stdat(ptr, val, _AMO_ST_AND);
+}
+
+static inline void amo_stdat_umax(uint64_t *ptr, uint64_t val) {
+  __builtin_amo_stdat(ptr, val, _AMO_ST_UMAX);
+}
+
+static inline void amo_stdat_umin(uint64_t *ptr, uint64_t val) {
+  __builtin_amo_stdat(ptr, val, _AMO_ST_UMIN);
+}
+
+/* 64-bit signed AMO store operations */
+static inline void amo_stdat_sadd(int64_t *ptr, int64_t val) {
+  __builtin_amo_stdat_s(ptr, val, _AMO_ST_ADD);
+}
+
+static inline void amo_stdat_smax(int64_t *ptr, int64_t val) {
+  __builtin_amo_stdat_s(ptr, val, _AMO_ST_SMAX);
+}
+
+static inline void amo_stdat_smin(int64_t *ptr, int64_t val) {
+  __builtin_amo_stdat_s(ptr, val, _AMO_ST_SMIN);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _AMO_H */