@smake/eigen 1.0.2 → 1.1.1

package/eigen/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h

@@ -0,0 +1,901 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2020 Everton Constantino (everton.constantino@ibm.com)
+// Copyright (C) 2021 Chip Kerchner (chip.kerchner@ibm.com)
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
+#define EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
+
+// If using dynamic dispatch, set the CPU target.
+#if defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH)
+#pragma GCC push_options
+#pragma GCC target("cpu=power10,htm")
+#endif
+
+#ifdef __has_builtin
+#if !__has_builtin(__builtin_vsx_assemble_pair)
+#define __builtin_vsx_assemble_pair __builtin_mma_assemble_pair
+#endif
+#if !__has_builtin(__builtin_vsx_disassemble_pair)
+#define __builtin_vsx_disassemble_pair __builtin_mma_disassemble_pair
+#endif
+#endif
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+#include "MatrixProductMMAbfloat16.h"
+
+namespace Eigen {
+
+namespace internal {
+
+#define accColsC (accCols / 2)
+
+EIGEN_ALWAYS_INLINE void bsetzeroMMA(__vector_quad* acc) { __builtin_mma_xxsetaccz(acc); }
+
+template <typename DataMapper, typename Packet, bool full>
+EIGEN_ALWAYS_INLINE void storeAccumulator(Index i, const DataMapper& data, const Packet& alpha, const Index elements,
+                                          __vector_quad* acc) {
+  PacketBlock<Packet, 4> result;
+  __builtin_mma_disassemble_acc(&result.packet, acc);
+
+  PacketBlock<Packet, 4> tRes;
+  if (full) {
+    EIGEN_UNUSED_VARIABLE(elements);
+    bload<DataMapper, Packet, 0, ColMajor, false, 4>(tRes, data, i, 0);
+    bscale<Packet, 4>(tRes, result, alpha);
+    bstore<DataMapper, Packet, 4>(tRes, data, i);
+  } else {
+    bload_partial<DataMapper, Packet, 0, false, 4>(tRes, data, i, elements);
+    bscale<Packet, 4>(tRes, result, alpha);
+    bstore_partial<DataMapper, Packet, 4>(tRes, data, i, elements);
+  }
+}
+
+template <typename DataMapper, typename Packet, typename Packetc, const Index accCols, const Index accCols2>
+EIGEN_ALWAYS_INLINE void storeComplexAccumulator(Index i, const DataMapper& data, const Packet& alphaReal,
+                                                 const Packet& alphaImag, const Packet& pMask, __vector_quad* accReal,
+                                                 __vector_quad* accImag) {
+  constexpr bool full = (accCols2 > accColsC);
+  PacketBlock<Packet, 4> resultReal, resultImag;
+  __builtin_mma_disassemble_acc(&resultReal.packet, accReal);
+  __builtin_mma_disassemble_acc(&resultImag.packet, accImag);
+
+  PacketBlock<Packetc, 8> tRes;
+  bload<DataMapper, Packetc, accColsC, ColMajor, true, 4, full>(tRes, data, i, 0);
+
+  PacketBlock<Packet, 4> taccReal, taccImag;
+  bscalec<Packet, 4, (accCols != accCols2)>(resultReal, resultImag, alphaReal, alphaImag, taccReal, taccImag, pMask);
+
+  PacketBlock<Packetc, 4> acc1, acc2;
+  bcouple<Packet, Packetc, 4, full>(taccReal, taccImag, tRes, acc1, acc2);
+
+  bstore<DataMapper, Packetc, 4>(acc1, data, i);
+  if (full) {
+    bstore<DataMapper, Packetc, 4>(acc2, data, i + accColsC);
+  }
+}
+
+// Defaults to float32, since Eigen still supports C++03 we can't use default template arguments
+template <typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
+EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const RhsPacket& a, const LhsPacket& b) {
+  if (NegativeAccumulate) {
+    __builtin_mma_xvf32gernp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
+  } else {
+    __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
+  }
+}
+
+template <typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
+EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const __vector_pair& a, const Packet2d& b) {
+  if (NegativeAccumulate) {
+    __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b);
+  } else {
+    __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b);
+  }
+}
+
+template <typename Packet, typename RhsPacket, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
+EIGEN_ALWAYS_INLINE void pgercMMA(__vector_quad* accReal, __vector_quad* accImag, const Packet& lhsV, Packet& lhsVi,
+                                  const RhsPacket& rhsV, RhsPacket& rhsVi) {
+  pgerMMA<Packet, RhsPacket, false>(accReal, rhsV, lhsV);
+  if (LhsIsReal) {
+    pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi, lhsV);
+    EIGEN_UNUSED_VARIABLE(lhsVi);
+  } else {
+    if (!RhsIsReal) {
+      pgerMMA<Packet, RhsPacket, ConjugateLhs == ConjugateRhs>(accReal, rhsVi, lhsVi);
+      pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi, lhsV);
+    } else {
+      EIGEN_UNUSED_VARIABLE(rhsVi);
+    }
+    pgerMMA<Packet, RhsPacket, ConjugateLhs>(accImag, rhsV, lhsVi);
+  }
+}
+
+// This is necessary because ploadRhs for double returns a pair of vectors when MMA is enabled.
+template <typename Packet>
+EIGEN_ALWAYS_INLINE Packet ploadRhs(const __UNPACK_TYPE__(Packet) * rhs) {
+  return ploadu<Packet>(rhs);
+}
+
+template <typename Scalar, typename Packet>
+EIGEN_ALWAYS_INLINE void ploadRhsMMA(const Scalar* rhs, Packet& rhsV) {
+  rhsV = ploadRhs<Packet>(rhs);
+}
+
+template <>
+EIGEN_ALWAYS_INLINE void ploadRhsMMA(const double* rhs, __vector_pair& rhsV) {
+#if EIGEN_COMP_LLVM
+  __builtin_vsx_assemble_pair(
+      &rhsV, reinterpret_cast<__vector unsigned char>(ploadRhs<Packet2d>(rhs + (sizeof(Packet2d) / sizeof(double)))),
+      reinterpret_cast<__vector unsigned char>(ploadRhs<Packet2d>(rhs)));
+#else
+  rhsV = *reinterpret_cast<__vector_pair*>(const_cast<double*>(rhs));
+#endif
+}
+
+EIGEN_ALWAYS_INLINE void ploadLhsMMA(const double* lhs, __vector_pair& lhsV) { ploadRhsMMA(lhs, lhsV); }
+
+#define GEMM_MULTIPLE_COLS
+
+// Disable in GCC until unnecessary register moves are fixed
+// #if (EIGEN_COMP_LLVM || (__GNUC__ >= 11))
+#if EIGEN_COMP_LLVM
+#define VECTOR_PAIR_LOADS_LHS
+#endif
+
+// PEEL_MMA loop factor.
+#ifdef GEMM_MULTIPLE_COLS
+#define PEEL_MMA 8
+#else
+// Register spillage with GCC12+
+#if EIGEN_COMP_LLVM || (__GNUC__ < 12) || defined(VECTOR_PAIR_LOADS_LHS)
+#define PEEL_MMA 7
+#else
+#define PEEL_MMA 6
+#endif
+#endif
+
+#define MICRO_MMA_UNROLL(func) func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)
+
+#define MICRO_MMA_WORK(func, type, peel)                                                                        \
+  if (accItr == 1) {                                                                                            \
+    func(0, type, peel, 0, 0) func(1, type, peel, 1, 0) func(2, type, peel, 2, 0) func(3, type, peel, 3, 0)     \
+        func(4, type, peel, 4, 0) func(5, type, peel, 5, 0) func(6, type, peel, 6, 0) func(7, type, peel, 7, 0) \
+  } else if (accItr == 2) {                                                                                     \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 1, 0) func(3, type, peel, 1, 1)     \
+        func(4, type, peel, 2, 0) func(5, type, peel, 2, 1) func(6, type, peel, 3, 0) func(7, type, peel, 3, 1) \
+  } else {                                                                                                      \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 0, 2) func(3, type, peel, 0, 3)     \
+        func(4, type, peel, 1, 0) func(5, type, peel, 1, 1) func(6, type, peel, 1, 2) func(7, type, peel, 1, 3) \
+  }
+
+#define MICRO_MMA_WORK_ONE(iter, type, peel, left, right)                        \
+  if (unroll_factor > left) {                                                    \
+    pgerMMA<Packet, type, false>(&accZero##iter, rhsV##right[peel], lhsV##left); \
+  }
+
+#ifdef VECTOR_PAIR_LOADS_LHS
+#define MICRO_MMA_WORK_TWO(iter, type, peel, left, right)                                          \
+  if (unroll_factor > left) {                                                                      \
+    pgerMMA<Packet, type, false>(&accZero##iter, rhsV##right[peel], lhsV2##left.packet[peel & 1]); \
+  }
+
+#define MICRO_MMA_LOAD1_TWO(lhs_ptr, left)                                                        \
+  if (unroll_factor > left) {                                                                     \
+    if (MICRO_NORMAL(left)) {                                                                     \
+      ploadLhsMMA(reinterpret_cast<const double*>(lhs_ptr##left), plhsV##left);                   \
+      __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&lhsV2##left.packet), &plhsV##left); \
+      lhs_ptr##left += accCols * 2;                                                               \
+    } else {                                                                                      \
+      lhsV2##left.packet[0] = ploadLhs<Packet>(lhs_ptr##left);                                    \
+      lhsV2##left.packet[1] = ploadLhs<Packet>(lhs_ptr##left + accCols2);                         \
+      lhs_ptr##left += accCols2 * 2;                                                              \
+      EIGEN_UNUSED_VARIABLE(plhsV##left);                                                         \
+    }                                                                                             \
+  } else {                                                                                        \
+    EIGEN_UNUSED_VARIABLE(lhsV2##left);                                                           \
+    EIGEN_UNUSED_VARIABLE(plhsV##left);                                                           \
+  }
+
+#define MICRO_MMA_LOAD_TWO(left) MICRO_MMA_LOAD1_TWO(lhs_ptr, left)
+#endif
+
+#define MICRO_MMA_UNROLL_ITER(func, val)                       \
+  func(val, 0) if (accItr > 1) {                               \
+    func(val, 1) if (accItr > 2) { func(val, 2) func(val, 3) } \
+  }
+
+#define MICRO_MMA_LOAD_ONE_RHS1(peel, right) ploadRhsMMA(rhs_ptr##right + (accRows * peel), rhsV##right[peel]);
+
+#define MICRO_MMA_LOAD_ONE_RHS(peel) MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_ONE_RHS1, peel)
+
+#define MICRO_MMA_TYPE_PEEL(funcw, funcl, type, peel)              \
+  if (PEEL_MMA > peel) {                                           \
+    Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \
+    MICRO_MMA_LOAD_ONE_RHS(peel)                                   \
+    MICRO_MMA_UNROLL(funcl)                                        \
+    MICRO_MMA_WORK(funcw, type, peel)                              \
+  }
+
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type)                                                  \
+  type rhsV0[8], rhsV1[(accItr > 1) ? 8 : 1], rhsV2[(accItr > 2) ? 8 : 1], rhsV3[(accItr > 2) ? 8 : 1]; \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 0)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 1)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 2)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 3)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 4)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 5)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 6) MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 7)
+#else
+#define MICRO_MMA_LOAD_TWO_RHS(peel1, right)                                                      \
+  ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr##right + (accRows * peel1)), prhsV##peel1); \
+  __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsV##right[peel1]), &prhsV##peel1);
+
+#define MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, peel1, peel2)           \
+  if (PEEL_MMA > peel2) {                                                                  \
+    PacketBlock<Packet, 2> lhsV20, lhsV21, lhsV22, lhsV23, lhsV24, lhsV25, lhsV26, lhsV27; \
+    __vector_pair plhsV0, plhsV1, plhsV2, plhsV3, plhsV4, plhsV5, plhsV6, plhsV7;          \
+    if (sizeof(type) == 16) {                                                              \
+      MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_TWO_RHS, peel1)                                 \
+    } else {                                                                               \
+      EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                                 \
+      MICRO_MMA_LOAD_ONE_RHS(peel1)                                                        \
+      MICRO_MMA_LOAD_ONE_RHS(peel2)                                                        \
+    }                                                                                      \
+    MICRO_MMA_UNROLL(funcl2)                                                               \
+    MICRO_MMA_WORK(funcw2, type, peel1)                                                    \
+    MICRO_MMA_WORK(funcw2, type, peel2)                                                    \
+  } else {                                                                                 \
+    EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                                   \
+    MICRO_MMA_TYPE_PEEL(funcw1, funcl1, type, peel1)                                       \
+  }
+
+#define MICRO_MMA_UNROLL_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type)                               \
+  type rhsV0[8], rhsV1[(accItr > 1) ? 8 : 1], rhsV2[(accItr > 2) ? 8 : 1], rhsV3[(accItr > 2) ? 8 : 1]; \
+  __vector_pair prhsV0, prhsV2, prhsV4, prhsV6;                                                         \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 0, 1)                                      \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 2, 3)                                      \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 4, 5)                                      \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 6, 7)
+#endif
+
+#define MICRO_MMA_UNROLL_TYPE_ONE(funcw, funcl, type) \
+  type rhsV0[1], rhsV1[1], rhsV2[1], rhsV3[1];        \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 0)
+
+#define MICRO_MMA_UPDATE_RHS1(size, right) rhs_ptr##right += (accRows * size);
+
+#define MICRO_MMA_UPDATE_RHS(size) MICRO_MMA_UNROLL_ITER(MICRO_MMA_UPDATE_RHS1, size)
+
+#define MICRO_MMA_UNROLL_TYPE(MICRO_MMA_TYPE, size)             \
+  MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, RhsPacket) \
+  MICRO_MMA_UPDATE_RHS(size)
+
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_MMA_ONE_PEEL MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_PEEL, PEEL_MMA)
+#else
+#define MICRO_MMA_UNROLL_TYPE2(MICRO_MMA_TYPE, size)                                                    \
+  MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, MICRO_MMA_WORK_TWO, MICRO_MMA_LOAD_TWO, RhsPacket) \
+  MICRO_MMA_UPDATE_RHS(size)
+
+#define MICRO_MMA_ONE_PEEL MICRO_MMA_UNROLL_TYPE2(MICRO_MMA_UNROLL_TYPE_PEEL2, PEEL_MMA)
+#endif
+
+#define MICRO_MMA_ONE MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_ONE, 1)
+
+#define MICRO_MMA_DST_PTR_ONE(iter)       \
+  if (unroll_factor * accItr > iter) {    \
+    bsetzeroMMA(&accZero##iter);          \
+  } else {                                \
+    EIGEN_UNUSED_VARIABLE(accZero##iter); \
+  }
+
+#define MICRO_MMA_DST_PTR MICRO_MMA_UNROLL(MICRO_MMA_DST_PTR_ONE)
+
+#define MICRO_MMA_SRC_PTR MICRO_MMA_UNROLL(MICRO_SRC_PTR_ONE)
+
+#define MICRO_MMA_PREFETCH MICRO_MMA_UNROLL(MICRO_PREFETCH_ONE)
+
+#define MICRO_MMA_STORE_ONE(iter, left, right)                                                                 \
+  if (unroll_factor > left) {                                                                                  \
+    storeAccumulator<DataMapper, Packet, MICRO_NORMAL_PARTIAL(left)>(row + left * accCols, res##right, pAlpha, \
+                                                                     accCols2, &accZero##iter);                \
+  }
+
+#define MICRO_MMA_ITER_UNROLL(func)                                                                                 \
+  if (accItr == 1) {                                                                                                \
+    func(0, 0, 0) func(1, 1, 0) func(2, 2, 0) func(3, 3, 0) func(4, 4, 0) func(5, 5, 0) func(6, 6, 0) func(7, 7, 0) \
+  } else if (accItr == 2) {                                                                                         \
+    func(0, 0, 0) func(1, 0, 1) func(2, 1, 0) func(3, 1, 1) func(4, 2, 0) func(5, 2, 1) func(6, 3, 0) func(7, 3, 1) \
+  } else {                                                                                                          \
+    func(0, 0, 0) func(1, 0, 1) func(2, 0, 2) func(3, 0, 3) func(4, 1, 0) func(5, 1, 1) func(6, 1, 2) func(7, 1, 3) \
+  }
+
+#define MICRO_MMA_STORE MICRO_MMA_ITER_UNROLL(MICRO_MMA_STORE_ONE)
+
+#define MICRO_MMA_EXTRA_ROWS(right)                                                                           \
+  gemm_extra_row<Scalar, Packet, DataMapper, accRows, accCols>(                                               \
+      res3##right, blockA, rhs_base + right * accRows * strideB, depth, strideA, offsetA, strideB, row, rows, \
+      remaining_rows, pAlpha, pMask);
+
+#define MICRO_MMA_EXTRA_ROWS1(val, right) MICRO_MMA_EXTRA_ROWS(right);
+
+template <int unroll_factor, typename Scalar, typename Packet, typename RhsPacket, typename DataMapper,
+          const Index accRows, const Index accCols, bool full, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemm_unrolled_MMA_iteration(const DataMapper& res0, const DataMapper& res1,
+                                                     const DataMapper& res2, const DataMapper& res3,
+                                                     const Scalar* lhs_base, const Scalar* rhs_base, Index depth,
+                                                     Index strideA, Index strideB, Index offsetA, Index& row,
+                                                     const Packet& pAlpha, Index accCols2) {
+  const Scalar *rhs_ptr0 = rhs_base, *rhs_ptr1 = NULL, *rhs_ptr2 = NULL, *rhs_ptr3 = NULL;
+  const Scalar *lhs_ptr0 = NULL, *lhs_ptr1 = NULL, *lhs_ptr2 = NULL, *lhs_ptr3 = NULL, *lhs_ptr4 = NULL,
+               *lhs_ptr5 = NULL, *lhs_ptr6 = NULL, *lhs_ptr7 = NULL;
+  __vector_quad accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7;
+
+  if (accItr > 1) {
+    rhs_ptr1 = rhs_base + (accRows * strideB);
+  } else {
+    EIGEN_UNUSED_VARIABLE(strideB);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr1);
+    EIGEN_UNUSED_VARIABLE(res1);
+  }
+  if (accItr > 2) {
+    rhs_ptr2 = rhs_base + (2 * accRows * strideB);
+    rhs_ptr3 = rhs_base + (3 * accRows * strideB);
+  } else {
+    EIGEN_UNUSED_VARIABLE(rhs_ptr2);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr3);
+    EIGEN_UNUSED_VARIABLE(res2);
+    EIGEN_UNUSED_VARIABLE(res3);
+  }
+
+  MICRO_MMA_SRC_PTR
+  MICRO_MMA_DST_PTR
+
+  Index k = 0, depth2 = depth - PEEL_MMA;
+  for (; k <= depth2; k += PEEL_MMA) {
+    EIGEN_POWER_PREFETCH(rhs_ptr);
+    MICRO_MMA_PREFETCH
+    MICRO_MMA_ONE_PEEL
+  }
+  for (; k < depth; k++) {
+    MICRO_MMA_ONE
+  }
+  MICRO_MMA_STORE
+
+  MICRO_UPDATE
+}
+
+#define MICRO_MMA_UNROLL_ITER2(N, M)                                                                                 \
+  gemm_unrolled_MMA_iteration<N + (M ? 1 : 0), Scalar, Packet, RhsPacket, DataMapper, accRows, accCols, !M, accItr>( \
+      res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, strideB, offsetA, row, pAlpha,                 \
+      M ? remaining_rows : accCols);                                                                                 \
+  if (M) return;
+
+#define MICRO_MMA_ROWS(n)             \
+  while (row + n * accCols <= rows) { \
+    MICRO_MMA_UNROLL_ITER2(n, 0);     \
+  }
+
+template <typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows,
+          const Index accCols, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemmMMA_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index depth,
+                                      Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows,
+                                      Index remaining_rows, const Packet& pAlpha, const Packet& pMask) {
+  const DataMapper res30 = res.getSubMapper(0, col);
+  const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows * 1) : res30;
+  const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows * 2) : res30;
+  const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows * 3) : res30;
+
+  const Scalar* rhs_base = blockB + col * strideB + accRows * offsetB;
+  const Scalar* lhs_base = blockA + accCols * offsetA;
+  Index row = 0;
+
+#define MAX_MMA_UNROLL 7
+
+#if MAX_MMA_UNROLL < 2
+  if (1) {
+#elif MAX_MMA_UNROLL < 4
+  if (accItr <= 2) {
+#else
+  if (accItr == 1) {
+#endif
+    MICRO_MMA_ROWS(MAX_MMA_UNROLL);
+  } else if (accItr == 2) {
+    MICRO_MMA_ROWS(4);
+  } else {
+    MICRO_MMA_ROWS(2);
+  }
+  switch ((rows - row) / accCols) {
+#if MAX_MMA_UNROLL > 7
+    case 7:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 7)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 6
+    case 6:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 6)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 5
+    case 5:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 5)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 4
+    case 4:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 4)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 3
+    case 3:
+      if (accItr <= 2) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 3)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 2
+    case 2:
+      if (accItr <= 2) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 2)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 1
+    case 1:
+      MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 1)
+      break;
+#endif
+    default:
+      break;
+  }
+#undef MAX_MMA_UNROLL
+
+  if (remaining_rows > 0) {
+    MICRO_MMA_UNROLL_ITER(MICRO_MMA_EXTRA_ROWS1, 0)
+  }
+}
+
+#define MICRO_MMA_COLS(n)                                                                                          \
+  for (; col + n * accRows <= cols; col += n * accRows) {                                                          \
+    gemmMMA_cols<Scalar, Packet, RhsPacket2, DataMapper, accRows, accCols, n>(                                     \
+        res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlpha, pMask); \
+  }
+
+template <typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows,
+          const Index accCols>
+void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols,
+             Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
+  const Index remaining_rows = rows % accCols;
+
+  if (strideA == -1) strideA = depth;
+  if (strideB == -1) strideB = depth;
+
+  const Packet pAlpha = pset1<Packet>(alpha);
+  const Packet pMask = bmask<Packet>(remaining_rows);
+
+  typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2;
+
+  Index col = 0;
+#ifdef GEMM_MULTIPLE_COLS
+  MICRO_MMA_COLS(4);
+  MICRO_MMA_COLS(2);
+#endif
+  MICRO_MMA_COLS(1);
+
+  if (col != cols) {
+    gemm_extra_cols<Scalar, Packet, DataMapper, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB,
+                                                         col, rows, cols, remaining_rows, pAlpha, pMask);
+  }
+}
+
+#define advanceRows ((LhsIsReal) ? 1 : 2)
+#define advanceCols ((RhsIsReal) ? 1 : 2)
+
+// PEEL_COMPLEX_MMA loop factor.
+#ifdef GEMM_MULTIPLE_COLS
+#define PEEL_COMPLEX_MMA 4
+#else
+#define PEEL_COMPLEX_MMA 3
+#endif
+
+#define MICRO_COMPLEX_MMA_UNROLL(func) func(0) func(1) func(2) func(3)
+
+#define MICRO_COMPLEX_MMA_WORK(func, type, peel)                                                            \
+  if (accItr == 1) {                                                                                        \
+    func(0, type, peel, 0, 0) func(1, type, peel, 1, 0) func(2, type, peel, 2, 0) func(3, type, peel, 3, 0) \
+  } else if (accItr == 2) {                                                                                 \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 1, 0) func(3, type, peel, 1, 1) \
+  } else {                                                                                                  \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 0, 2) func(3, type, peel, 0, 3) \
+  }
+
+#define MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel, left, right)                                        \
+  if (unroll_factor > left) {                                                                            \
+    pgercMMA<Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(                            \
+        &accReal##iter, &accImag##iter, lhsV##left, lhsVi##left, rhsV##right[peel], rhsVi##right[peel]); \
+  }
+
+#ifdef VECTOR_PAIR_LOADS_LHS
+#define MICRO_COMPLEX_MMA_WORK_TWO(iter, type, peel, left, right)                                    \
+  if (unroll_factor > left) {                                                                        \
+    pgercMMA<Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(                        \
+        &accReal##iter, &accImag##iter, lhsV2##left.packet[peel & 1], lhsVi2##left.packet[peel & 1], \
+        rhsV##right[peel], rhsVi##right[peel]);                                                      \
+  }
+
+#define MICRO_COMPLEX_MMA_LOAD1_TWO(lhs_ptr, left)                                                  \
+  if (!LhsIsReal && (unroll_factor > left)) {                                                       \
+    if (MICRO_NORMAL(left)) {                                                                       \
+      ploadLhsMMA(reinterpret_cast<const double*>(lhs_ptr_real##left + imag_delta), plhsVi##left);  \
+      __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&lhsVi2##left.packet), &plhsVi##left); \
+    } else {                                                                                        \
+      lhsVi2##left.packet[0] = ploadLhs<Packet>(lhs_ptr_real##left + imag_delta2);                  \
+      lhsVi2##left.packet[1] = ploadLhs<Packet>(lhs_ptr_real##left + imag_delta2 + accCols2);       \
+      EIGEN_UNUSED_VARIABLE(plhsVi##left);                                                          \
+    }                                                                                               \
+  } else {                                                                                          \
+    EIGEN_UNUSED_VARIABLE(lhsVi2##left);                                                            \
+    EIGEN_UNUSED_VARIABLE(plhsVi##left);                                                            \
+  }                                                                                                 \
+  MICRO_MMA_LOAD1_TWO(lhs_ptr_real, left)
+
+#define MICRO_COMPLEX_MMA_LOAD_TWO(left) MICRO_COMPLEX_MMA_LOAD1_TWO(lhs_ptr, left)
+#endif
+
+#define MICRO_COMPLEX_MMA_LOAD_RHS1(peel, right)                             \
+  ploadRhsMMA(rhs_ptr_real##right + (accRows * peel), rhsV##right[peel]);    \
+  if (!RhsIsReal) {                                                          \
+    ploadRhsMMA(rhs_ptr_imag##right + (accRows * peel), rhsVi##right[peel]); \
+  }
+
+#define MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel) MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_RHS1, peel)
+
+#define MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, peel) \
+  if (PEEL_COMPLEX_MMA > peel) {                              \
+    Packet lhsV0, lhsV1, lhsV2, lhsV3;                        \
+    Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3;                    \
+    MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel)                      \
+    MICRO_COMPLEX_MMA_UNROLL(funcl)                           \
+    MICRO_COMPLEX_MMA_WORK(funcw, type, peel)                 \
+  }
+
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type)                                                      \
+  type rhsV0[4], rhsVi0[4], rhsV1[(accItr > 1) ? 4 : 1], rhsVi1[(accItr > 1) ? 4 : 1], rhsV2[(accItr > 2) ? 4 : 1], \
+      rhsVi2[(accItr > 2) ? 4 : 1], rhsV3[(accItr > 2) ? 4 : 1], rhsVi3[(accItr > 2) ? 4 : 1];                      \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 0)                                                                \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 1)                                                                \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 2) MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 3)
+#else
+#define MICRO_COMPLEX_MMA_LOAD_TWO_RHS(peel1, right)                                                      \
+  ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr_real##right + (accRows * peel1)), prhsV##peel1);    \
+  __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsV##right[peel1]), &prhsV##peel1);            \
+  if (!RhsIsReal) {                                                                                       \
+    ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr_imag##right + (accRows * peel1)), prhsVi##peel1); \
+    __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsVi##right[peel1]), &prhsVi##peel1);        \
+  } else {                                                                                                \
+    EIGEN_UNUSED_VARIABLE(prhsVi##peel1);                                                                 \
+  }
+
+#define MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, peel1, peel2) \
+  if (PEEL_COMPLEX_MMA > peel2) {                                                        \
+    PacketBlock<Packet, 2> lhsV20, lhsV21, lhsV22, lhsV23;                               \
+    PacketBlock<Packet, 2> lhsVi20, lhsVi21, lhsVi22, lhsVi23;                           \
+    __vector_pair plhsV0, plhsV1, plhsV2, plhsV3;                                        \
+    __vector_pair plhsVi0, plhsVi1, plhsVi2, plhsVi3;                                    \
+    if (sizeof(type) == 16) {                                                            \
+      MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_TWO_RHS, peel1)                       \
+    } else {                                                                             \
+      EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                               \
+      EIGEN_UNUSED_VARIABLE(prhsVi##peel1);                                              \
+      MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel1);                                             \
+      MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel2);                                             \
+    }                                                                                    \
+    MICRO_COMPLEX_MMA_UNROLL(funcl2)                                                     \
+    MICRO_COMPLEX_MMA_WORK(funcw2, type, peel1)                                          \
+    MICRO_COMPLEX_MMA_WORK(funcw2, type, peel2)                                          \
+  } else {                                                                               \
+    EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                                 \
+    EIGEN_UNUSED_VARIABLE(prhsVi##peel1);                                                \
+    MICRO_COMPLEX_MMA_TYPE_PEEL(funcw1, funcl1, type, peel1)                             \
+  }
+
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type)                                   \
+  type rhsV0[4], rhsVi0[4], rhsV1[(accItr > 1) ? 4 : 1], rhsVi1[(accItr > 1) ? 4 : 1], rhsV2[(accItr > 2) ? 4 : 1], \
+      rhsVi2[(accItr > 2) ? 4 : 1], rhsV3[(accItr > 2) ? 4 : 1], rhsVi3[(accItr > 2) ? 4 : 1];                      \
+  __vector_pair prhsV0, prhsV2;                                                                                     \
+  __vector_pair prhsVi0, prhsVi2;                                                                                   \
+  MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 0, 1)                                          \
+  MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 2, 3)
+#endif
+
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(funcw, funcl, type)                              \
+  type rhsV0[1], rhsVi0[1], rhsV1[1], rhsVi1[1], rhsV2[1], rhsVi2[1], rhsV3[1], rhsVi3[1]; \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 0)
+
+#define MICRO_COMPLEX_MMA_UPDATE_RHS1(size, right) \
+  rhs_ptr_real##right += (accRows * size);         \
+  if (!RhsIsReal) rhs_ptr_imag##right += (accRows * size);
+
+#define MICRO_COMPLEX_MMA_UPDATE_RHS(size) MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_UPDATE_RHS1, size)
+
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_TYPE, size)                     \
+  MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, RhsPacket) \
+  MICRO_COMPLEX_MMA_UPDATE_RHS(size);
+
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_COMPLEX_MMA_ONE_PEEL MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL, PEEL_COMPLEX_MMA)
+#else
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE2(MICRO_COMPLEX_MMA_TYPE, size)                                     \
+  MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, MICRO_COMPLEX_MMA_WORK_TWO, \
+                         MICRO_COMPLEX_MMA_LOAD_TWO, RhsPacket)                                          \
+  MICRO_COMPLEX_MMA_UPDATE_RHS(size);
+
+#define MICRO_COMPLEX_MMA_ONE_PEEL MICRO_COMPLEX_MMA_UNROLL_TYPE2(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL2, PEEL_COMPLEX_MMA)
+#endif
+
+#define MICRO_COMPLEX_MMA_ONE MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE, 1)
+
+#define MICRO_COMPLEX_MMA_DST_PTR_ONE(iter) \
+  if (unroll_factor * accItr > iter) {      \
+    bsetzeroMMA(&accReal##iter);            \
+    bsetzeroMMA(&accImag##iter);            \
+  } else {                                  \
+    EIGEN_UNUSED_VARIABLE(accReal##iter);   \
+    EIGEN_UNUSED_VARIABLE(accImag##iter);   \
+  }
+
+#define MICRO_COMPLEX_MMA_DST_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_DST_PTR_ONE)
+
+#define MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE)
+
+#define MICRO_COMPLEX_MMA_PREFETCH MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_PREFETCH_ONE)
+
+#define MICRO_COMPLEX_MMA_STORE_ONE(iter, left, right)                                                                 \
+  if (unroll_factor > left) {                                                                                          \
+    storeComplexAccumulator<DataMapper, Packet, Packetc, accCols, (unroll_factor != (left + 1)) ? accCols : accCols2>( \
+        row + left * accCols, res##right, pAlphaReal, pAlphaImag, pMask, &accReal##iter, &accImag##iter);              \
+  }
+
+#define MICRO_COMPLEX_MMA_ITER_UNROLL(func)                 \
+  if (accItr == 1) {                                        \
+    func(0, 0, 0) func(1, 1, 0) func(2, 2, 0) func(3, 3, 0) \
+  } else if (accItr == 2) {                                 \
+    func(0, 0, 0) func(1, 0, 1) func(2, 1, 0) func(3, 1, 1) \
+  } else {                                                  \
+    func(0, 0, 0) func(1, 0, 1) func(2, 0, 2) func(3, 0, 3) \
+  }
+
+#define MICRO_COMPLEX_MMA_STORE MICRO_COMPLEX_MMA_ITER_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE)
+
+#define MICRO_COMPLEX_MMA_EXTRA_ROWS(right)                                                                            \
+  gemm_complex_extra_row<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, \
+                         RhsIsReal>(res3##right, blockA, rhs_base + right * accRows * (RhsIsReal ? 1 : 2) * strideB,   \
+                                    depth, strideA, offsetA, strideB, row, rows, remaining_rows, pAlphaReal,           \
+                                    pAlphaImag, pMask);
+
+#define MICRO_COMPLEX_MMA_EXTRA_ROWS1(val, right) MICRO_COMPLEX_MMA_EXTRA_ROWS(right);
+
+template <int unroll_factor, typename Scalar, typename Packet, typename Packetc, typename RhsPacket,
+          typename DataMapper, const Index accRows, const Index accCols, const Index accCols2, bool ConjugateLhs,
+          bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemm_complex_unrolled_MMA_iteration(const DataMapper& res0, const DataMapper& res1,
+                                                             const DataMapper& res2, const DataMapper& res3,
+                                                             const Scalar* lhs_base, const Scalar* rhs_base,
+                                                             Index depth, Index strideA, Index offsetA, Index strideB,
+                                                             Index& row, const Packet& pAlphaReal,
+                                                             const Packet& pAlphaImag, const Packet& pMask) {
+  const Scalar *rhs_ptr_real0 = rhs_base, *rhs_ptr_real1 = NULL, *rhs_ptr_real2 = NULL, *rhs_ptr_real3 = NULL;
+  const Scalar *rhs_ptr_imag0 = NULL, *rhs_ptr_imag1 = NULL, *rhs_ptr_imag2 = NULL, *rhs_ptr_imag3 = NULL;
+  const Index imag_delta = accCols * strideA;
+  const Index imag_delta2 = accCols2 * strideA;
+
+  if (!RhsIsReal) {
+    rhs_ptr_imag0 = rhs_base + accRows * strideB;
+  } else {
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag0);
+  }
+  if (accItr > 1) {
+    if (!RhsIsReal) {
+      rhs_ptr_real1 = rhs_base + (2 * accRows * strideB);
+      rhs_ptr_imag1 = rhs_base + (3 * accRows * strideB);
+    } else {
+      rhs_ptr_real1 = rhs_base + accRows * strideB;
+      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1);
+    }
+  } else {
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_real1);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1);
+    EIGEN_UNUSED_VARIABLE(res1);
+  }
+  if (accItr > 2) {
+    if (!RhsIsReal) {
+      rhs_ptr_real2 = rhs_base + (4 * accRows * strideB);
+      rhs_ptr_imag2 = rhs_base + (5 * accRows * strideB);
+      rhs_ptr_real3 = rhs_base + (6 * accRows * strideB);
+      rhs_ptr_imag3 = rhs_base + (7 * accRows * strideB);
+    } else {
+      rhs_ptr_real2 = rhs_base + (2 * accRows * strideB);
+      rhs_ptr_real3 = rhs_base + (3 * accRows * strideB);
+      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2);
+      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3);
+    }
+  } else {
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_real2);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_real3);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3);
+    EIGEN_UNUSED_VARIABLE(res2);
+    EIGEN_UNUSED_VARIABLE(res3);
+  }
+  const Scalar *lhs_ptr_real0 = NULL, *lhs_ptr_real1 = NULL;
+  const Scalar *lhs_ptr_real2 = NULL, *lhs_ptr_real3 = NULL;
+  __vector_quad accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3;
+
+  MICRO_COMPLEX_MMA_SRC_PTR
+  MICRO_COMPLEX_MMA_DST_PTR
+
+  Index k = 0, depth2 = depth - PEEL_COMPLEX_MMA;
+  for (; k <= depth2; k += PEEL_COMPLEX_MMA) {
+    EIGEN_POWER_PREFETCH(rhs_ptr_real);
+    if (!RhsIsReal) {
+      EIGEN_POWER_PREFETCH(rhs_ptr_imag);
+    }
+    MICRO_COMPLEX_MMA_PREFETCH
+    MICRO_COMPLEX_MMA_ONE_PEEL
+  }
+  for (; k < depth; k++) {
+    MICRO_COMPLEX_MMA_ONE
+  }
+  MICRO_COMPLEX_MMA_STORE
+
+  MICRO_COMPLEX_UPDATE
+}
+
+#define MICRO_COMPLEX_MMA_UNROLL_ITER2(N, M)                                                                           \
+  gemm_complex_unrolled_MMA_iteration<N + (M ? 1 : 0), Scalar, Packet, Packetc, RhsPacket, DataMapper, accRows,        \
+                                      accCols, M ? M : accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal,      \
+                                      accItr>(res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, offsetA, \
+                                              strideB, row, pAlphaReal, pAlphaImag, pMask);                            \
+  if (M) return;
+
+#define MICRO_COMPLEX_MMA_ROWS(n)         \
+  while (row + n * accCols <= rows) {     \
+    MICRO_COMPLEX_MMA_UNROLL_ITER2(n, 0); \
+  }
+
+template <typename Scalar, typename Packet, typename Packetc, typename RhsPacket, typename DataMapper,
+          const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal,
+          bool RhsIsReal, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemmMMA_complex_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB,
+                                              Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB,
+                                              Index col, Index rows, Index remaining_rows, const Packet& pAlphaReal,
+                                              const Packet& pAlphaImag, const Packet& pMask) {
+  const DataMapper res30 = res.getSubMapper(0, col);
+  const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows * 1) : res30;
+  const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows * 2) : res30;
+  const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows * 3) : res30;
+
+  const Scalar* rhs_base = blockB + advanceCols * col * strideB + accRows * offsetB;
+  const Scalar* lhs_base = blockA + accCols * offsetA;
+  Index row = 0;
+
+#define MAX_COMPLEX_MMA_UNROLL 4
+
+#if MAX_COMPLEX_MMA_UNROLL < 2
+  if (1) {
+#elif MAX_COMPLEX_MMA_UNROLL < 4
+  if (accItr <= 2) {
+#else
+  if (accItr == 1) {
+#endif
+    MICRO_COMPLEX_MMA_ROWS(MAX_COMPLEX_MMA_UNROLL);
+  } else if (accItr == 2) {
+    MICRO_COMPLEX_MMA_ROWS(2);
+  } else {
+    MICRO_COMPLEX_MMA_ROWS(1);
+  }
+  switch ((rows - row) / accCols) {
+#if MAX_COMPLEX_MMA_UNROLL > 3
+    case 3:
+      if (accItr == 1) {
+        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 3)
+      }
+      break;
+#endif
+#if MAX_COMPLEX_MMA_UNROLL > 2
+    case 2:
+      if (accItr == 1) {
+        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 2)
+      }
+      break;
+#endif
+#if MAX_COMPLEX_MMA_UNROLL > 1
+    case 1:
+      if (accItr <= 2) {
+        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 1)
+      }
+      break;
+#endif
+    default:
+      break;
+  }
+#undef MAX_COMPLEX_MMA_UNROLL
+
+  if (remaining_rows > 0) {
+    MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_EXTRA_ROWS1, 0)
+  }
+}
+
+#define MICRO_COMPLEX_MMA_COLS(n)                                                                                      \
+  for (; col + n * accRows <= cols; col += n * accRows) {                                                              \
+    gemmMMA_complex_cols<Scalar, Packet, Packetc, RhsPacket2, DataMapper, accRows, accCols, ConjugateLhs,              \
+                         ConjugateRhs, LhsIsReal, RhsIsReal, n>(res, blockA, blockB, depth, strideA, offsetA, strideB, \
+                                                                offsetB, col, rows, remaining_rows, pAlphaReal,        \
+                                                                pAlphaImag, pMask);                                    \
+  }
+
+template <typename LhsScalar, typename RhsScalar, typename Scalarc, typename Scalar, typename Packet, typename Packetc,
+          typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols, bool ConjugateLhs,
+          bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
+void gemm_complexMMA(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc, Index rows, Index depth,
+                     Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
+  const Index remaining_rows = rows % accCols;
+
+  if (strideA == -1) strideA = depth;
+  if (strideB == -1) strideB = depth;
+
+  const Packet pAlphaReal = pset1<Packet>(alpha.real());
+  const Packet pAlphaImag = pset1<Packet>(alpha.imag());
+  const Packet pMask = bmask<Packet>(remaining_rows);
+
+  const Scalar* blockA = (Scalar*)blockAc;
+  const Scalar* blockB = (Scalar*)blockBc;
+
+  typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2;
+
+  Index col = 0;
+#ifdef GEMM_MULTIPLE_COLS
+  MICRO_COMPLEX_MMA_COLS(4);
+  MICRO_COMPLEX_MMA_COLS(2);
+#endif
+  MICRO_COMPLEX_MMA_COLS(1);
+
+  if (col != cols) {
+    gemm_complex_extra_cols<Scalar, Packet, Packetc, DataMapper, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
+                            RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols,
+                                       remaining_rows, pAlphaReal, pAlphaImag, pMask);
+  }
+}
+
+#undef accColsC
+#undef advanceRows
+#undef advanceCols
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#if defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH)
+#pragma GCC pop_options
+#endif
+
+#endif  // EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H