@smake/eigen 1.1.0 → 1.1.1

package/eigen/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h

@@ -11,619 +11,891 @@
 #ifndef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
 #define EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
 
-#pragma GCC target("cpu=power10")
+// If using dynamic dispatch, set the CPU target.
+#if defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH)
+#pragma GCC push_options
+#pragma GCC target("cpu=power10,htm")
+#endif
 
 #ifdef __has_builtin
 #if !__has_builtin(__builtin_vsx_assemble_pair)
 #define __builtin_vsx_assemble_pair __builtin_mma_assemble_pair
 #endif
+#if !__has_builtin(__builtin_vsx_disassemble_pair)
+#define __builtin_vsx_disassemble_pair __builtin_mma_disassemble_pair
+#endif
 #endif
 
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+#include "MatrixProductMMAbfloat16.h"
+
 namespace Eigen {
 
 namespace internal {
 
-template<typename Scalar, typename Packet>
-EIGEN_ALWAYS_INLINE void bsetzeroMMA(__vector_quad* acc)
-{
-  __builtin_mma_xxsetaccz(acc);
-}
+#define accColsC (accCols / 2)
+
+EIGEN_ALWAYS_INLINE void bsetzeroMMA(__vector_quad* acc) { __builtin_mma_xxsetaccz(acc); }
 
-template<typename DataMapper, typename Index, typename Packet, const Index accCols>
-EIGEN_ALWAYS_INLINE void storeAccumulator(Index i, Index j, const DataMapper& data, const Packet& alpha, __vector_quad* acc)
-{
+template <typename DataMapper, typename Packet, bool full>
+EIGEN_ALWAYS_INLINE void storeAccumulator(Index i, const DataMapper& data, const Packet& alpha, const Index elements,
+                                          __vector_quad* acc) {
   PacketBlock<Packet, 4> result;
   __builtin_mma_disassemble_acc(&result.packet, acc);
 
   PacketBlock<Packet, 4> tRes;
-  bload<DataMapper, Packet, Index, accCols, 0, ColMajor>(tRes, data, i, j);
-
-  bscale<Packet>(tRes, result, alpha);
-
-  data.template storePacketBlock<Packet, 4>(i, j, tRes);
+  if (full) {
+    EIGEN_UNUSED_VARIABLE(elements);
+    bload<DataMapper, Packet, 0, ColMajor, false, 4>(tRes, data, i, 0);
+    bscale<Packet, 4>(tRes, result, alpha);
+    bstore<DataMapper, Packet, 4>(tRes, data, i);
+  } else {
+    bload_partial<DataMapper, Packet, 0, false, 4>(tRes, data, i, elements);
+    bscale<Packet, 4>(tRes, result, alpha);
+    bstore_partial<DataMapper, Packet, 4>(tRes, data, i, elements);
+  }
 }
 
-template<typename DataMapper, typename Index, typename Packet, typename Packetc, const Index accColsC, int N>
-EIGEN_ALWAYS_INLINE void storeComplexAccumulator(Index i, Index j, const DataMapper& data, const Packet& alphaReal, const Packet& alphaImag, __vector_quad* accReal, __vector_quad* accImag)
-{
+template <typename DataMapper, typename Packet, typename Packetc, const Index accCols, const Index accCols2>
+EIGEN_ALWAYS_INLINE void storeComplexAccumulator(Index i, const DataMapper& data, const Packet& alphaReal,
+                                                 const Packet& alphaImag, const Packet& pMask, __vector_quad* accReal,
+                                                 __vector_quad* accImag) {
+  constexpr bool full = (accCols2 > accColsC);
   PacketBlock<Packet, 4> resultReal, resultImag;
   __builtin_mma_disassemble_acc(&resultReal.packet, accReal);
   __builtin_mma_disassemble_acc(&resultImag.packet, accImag);
 
   PacketBlock<Packetc, 8> tRes;
-  bload<DataMapper, Packetc, Index, accColsC, N, ColMajor>(tRes, data, i, j);
+  bload<DataMapper, Packetc, accColsC, ColMajor, true, 4, full>(tRes, data, i, 0);
 
-  PacketBlock<Packet,4> taccReal, taccImag;
-  bscalec<Packet,4>(resultReal, resultImag, alphaReal, alphaImag, taccReal, taccImag);
+  PacketBlock<Packet, 4> taccReal, taccImag;
+  bscalec<Packet, 4, (accCols != accCols2)>(resultReal, resultImag, alphaReal, alphaImag, taccReal, taccImag, pMask);
 
   PacketBlock<Packetc, 4> acc1, acc2;
-  bcouple<Packet, Packetc>(taccReal, taccImag, tRes, acc1, acc2);
+  bcouple<Packet, Packetc, 4, full>(taccReal, taccImag, tRes, acc1, acc2);
 
-  data.template storePacketBlock<Packetc, 4>(i + N*accColsC, j, acc1);
-  data.template storePacketBlock<Packetc, 4>(i + (N+1)*accColsC, j, acc2);
+  bstore<DataMapper, Packetc, 4>(acc1, data, i);
+  if (full) {
+    bstore<DataMapper, Packetc, 4>(acc2, data, i + accColsC);
+  }
 }
 
 // Defaults to float32, since Eigen still supports C++03 we can't use default template arguments
-template<typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
-EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const RhsPacket& a, const LhsPacket& b)
-{
-  if(NegativeAccumulate)
-  {
+template <typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
+EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const RhsPacket& a, const LhsPacket& b) {
+  if (NegativeAccumulate) {
     __builtin_mma_xvf32gernp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
   } else {
     __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
   }
 }
 
-template<typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
-EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const PacketBlock<Packet2d,2>& a, const Packet2d& b)
-{
-  __vector_pair* a0 = (__vector_pair *)(&a.packet[0]);
-  if(NegativeAccumulate)
-  {
-    __builtin_mma_xvf64gernp(acc, *a0, (__vector unsigned char)b);
-  } else {
-    __builtin_mma_xvf64gerpp(acc, *a0, (__vector unsigned char)b);
-  }
-}
-
-template<typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
-EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const __vector_pair& a, const Packet2d& b)
-{
-  if(NegativeAccumulate)
-  {
+template <typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
+EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const __vector_pair& a, const Packet2d& b) {
+  if (NegativeAccumulate) {
     __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b);
   } else {
     __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b);
   }
 }
 
-template<typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
-EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad*, const __vector_pair&, const Packet4f&)
-{
-  // Just for compilation
-}
-
-template<typename Scalar, typename Packet, typename RhsPacket, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
-EIGEN_ALWAYS_INLINE void pgercMMA(__vector_quad* accReal, __vector_quad* accImag, const Packet& lhsV, const Packet& lhsVi, const RhsPacket& rhsV, const RhsPacket& rhsVi)
-{
-  pgerMMA<Packet, RhsPacket, false>(accReal,  rhsV,  lhsV);
-  if(LhsIsReal) {
-    pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi,  lhsV);
+template <typename Packet, typename RhsPacket, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
+EIGEN_ALWAYS_INLINE void pgercMMA(__vector_quad* accReal, __vector_quad* accImag, const Packet& lhsV, Packet& lhsVi,
+                                  const RhsPacket& rhsV, RhsPacket& rhsVi) {
+  pgerMMA<Packet, RhsPacket, false>(accReal, rhsV, lhsV);
+  if (LhsIsReal) {
+    pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi, lhsV);
+    EIGEN_UNUSED_VARIABLE(lhsVi);
   } else {
-    if(!RhsIsReal) {
+    if (!RhsIsReal) {
       pgerMMA<Packet, RhsPacket, ConjugateLhs == ConjugateRhs>(accReal, rhsVi, lhsVi);
-      pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi,  lhsV);
+      pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi, lhsV);
     } else {
       EIGEN_UNUSED_VARIABLE(rhsVi);
     }
-    pgerMMA<Packet, RhsPacket, ConjugateLhs>(accImag,  rhsV, lhsVi);
+    pgerMMA<Packet, RhsPacket, ConjugateLhs>(accImag, rhsV, lhsVi);
   }
 }
 
 // This is necessary because ploadRhs for double returns a pair of vectors when MMA is enabled.
-template<typename Scalar, typename Packet>
-EIGEN_ALWAYS_INLINE void ploadRhsMMA(const Scalar* rhs, Packet& rhsV)
-{
-  rhsV = ploadRhs<Scalar, Packet>((const Scalar*)(rhs));
-} 
-
-template<>
-EIGEN_ALWAYS_INLINE void ploadRhsMMA<double, PacketBlock<Packet2d, 2> >(const double* rhs, PacketBlock<Packet2d, 2>& rhsV)
-{
-  rhsV.packet[0] = ploadRhs<double, Packet2d>((const double *)((Packet2d *)rhs      ));
-  rhsV.packet[1] = ploadRhs<double, Packet2d>((const double *)(((Packet2d *)rhs) + 1));
+template <typename Packet>
+EIGEN_ALWAYS_INLINE Packet ploadRhs(const __UNPACK_TYPE__(Packet) * rhs) {
+  return ploadu<Packet>(rhs);
+}
+
+template <typename Scalar, typename Packet>
+EIGEN_ALWAYS_INLINE void ploadRhsMMA(const Scalar* rhs, Packet& rhsV) {
+  rhsV = ploadRhs<Packet>(rhs);
 }
 
-template<>
-EIGEN_ALWAYS_INLINE void ploadRhsMMA<double, __vector_pair>(const double* rhs, __vector_pair& rhsV)
-{
+template <>
+EIGEN_ALWAYS_INLINE void ploadRhsMMA(const double* rhs, __vector_pair& rhsV) {
 #if EIGEN_COMP_LLVM
-  __builtin_vsx_assemble_pair(&rhsV,
-    (__vector unsigned char)(ploadRhs<double, Packet2d>((const double *)(((Packet2d *)rhs) + 1))),
-    (__vector unsigned char)(ploadRhs<double, Packet2d>((const double *)((Packet2d *)rhs      ))));
+  __builtin_vsx_assemble_pair(
+      &rhsV, reinterpret_cast<__vector unsigned char>(ploadRhs<Packet2d>(rhs + (sizeof(Packet2d) / sizeof(double)))),
+      reinterpret_cast<__vector unsigned char>(ploadRhs<Packet2d>(rhs)));
 #else
-  __asm__ ("lxvp %x0,%1" : "=wa" (rhsV) : "Y" (*rhs));
+  rhsV = *reinterpret_cast<__vector_pair*>(const_cast<double*>(rhs));
 #endif
 }
 
-template<>
-EIGEN_ALWAYS_INLINE void ploadRhsMMA(const float*, __vector_pair&)
-{
-  // Just for compilation
-}
+EIGEN_ALWAYS_INLINE void ploadLhsMMA(const double* lhs, __vector_pair& lhsV) { ploadRhsMMA(lhs, lhsV); }
+
+#define GEMM_MULTIPLE_COLS
+
+// Disable in GCC until unnecessary register moves are fixed
+// #if (EIGEN_COMP_LLVM || (__GNUC__ >= 11))
+#if EIGEN_COMP_LLVM
+#define VECTOR_PAIR_LOADS_LHS
+#endif
 
 // PEEL_MMA loop factor.
+#ifdef GEMM_MULTIPLE_COLS
+#define PEEL_MMA 8
+#else
+// Register spillage with GCC12+
+#if EIGEN_COMP_LLVM || (__GNUC__ < 12) || defined(VECTOR_PAIR_LOADS_LHS)
 #define PEEL_MMA 7
+#else
+#define PEEL_MMA 6
+#endif
+#endif
 
-#define MICRO_MMA_UNROLL(func) \
-  func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)
+#define MICRO_MMA_UNROLL(func) func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)
+
+#define MICRO_MMA_WORK(func, type, peel)                                                                        \
+  if (accItr == 1) {                                                                                            \
+    func(0, type, peel, 0, 0) func(1, type, peel, 1, 0) func(2, type, peel, 2, 0) func(3, type, peel, 3, 0)     \
+        func(4, type, peel, 4, 0) func(5, type, peel, 5, 0) func(6, type, peel, 6, 0) func(7, type, peel, 7, 0) \
+  } else if (accItr == 2) {                                                                                     \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 1, 0) func(3, type, peel, 1, 1)     \
+        func(4, type, peel, 2, 0) func(5, type, peel, 2, 1) func(6, type, peel, 3, 0) func(7, type, peel, 3, 1) \
+  } else {                                                                                                      \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 0, 2) func(3, type, peel, 0, 3)     \
+        func(4, type, peel, 1, 0) func(5, type, peel, 1, 1) func(6, type, peel, 1, 2) func(7, type, peel, 1, 3) \
+  }
 
-#define MICRO_MMA_LOAD_ONE(iter) \
-  if (unroll_factor > iter) { \
-    lhsV##iter = ploadLhs<Scalar, Packet>(lhs_ptr##iter); \
-    lhs_ptr##iter += accCols; \
-  } else { \
-    EIGEN_UNUSED_VARIABLE(lhsV##iter); \
+#define MICRO_MMA_WORK_ONE(iter, type, peel, left, right)                        \
+  if (unroll_factor > left) {                                                    \
+    pgerMMA<Packet, type, false>(&accZero##iter, rhsV##right[peel], lhsV##left); \
   }
 
-#define MICRO_MMA_WORK_ONE(iter, type, peel) \
-  if (unroll_factor > iter) { \
-    pgerMMA<Packet, type, false>(&accZero##iter, rhsV##peel, lhsV##iter); \
+#ifdef VECTOR_PAIR_LOADS_LHS
+#define MICRO_MMA_WORK_TWO(iter, type, peel, left, right)                                          \
+  if (unroll_factor > left) {                                                                      \
+    pgerMMA<Packet, type, false>(&accZero##iter, rhsV##right[peel], lhsV2##left.packet[peel & 1]); \
   }
 
-#define MICRO_MMA_TYPE_PEEL(func, func2, type, peel) \
-  if (PEEL_MMA > peel) { \
-    Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \
-    ploadRhsMMA<Scalar, type>(rhs_ptr + (accRows * peel), rhsV##peel); \
-    MICRO_MMA_UNROLL(func2); \
-    func(0,type,peel) func(1,type,peel) func(2,type,peel) func(3,type,peel) \
-    func(4,type,peel) func(5,type,peel) func(6,type,peel) func(7,type,peel) \
-  } else { \
-    EIGEN_UNUSED_VARIABLE(rhsV##peel); \
-  }
-
-#define MICRO_MMA_UNROLL_TYPE_PEEL(func, func2, type) \
-  type rhsV0, rhsV1, rhsV2, rhsV3, rhsV4, rhsV5, rhsV6, rhsV7, rhsV8, rhsV9; \
-  MICRO_MMA_TYPE_PEEL(func,func2,type,0); MICRO_MMA_TYPE_PEEL(func,func2,type,1); \
-  MICRO_MMA_TYPE_PEEL(func,func2,type,2); MICRO_MMA_TYPE_PEEL(func,func2,type,3); \
-  MICRO_MMA_TYPE_PEEL(func,func2,type,4); MICRO_MMA_TYPE_PEEL(func,func2,type,5); \
-  MICRO_MMA_TYPE_PEEL(func,func2,type,6); MICRO_MMA_TYPE_PEEL(func,func2,type,7); \
-  MICRO_MMA_TYPE_PEEL(func,func2,type,8); MICRO_MMA_TYPE_PEEL(func,func2,type,9);
-
-#define MICRO_MMA_UNROLL_TYPE_ONE(func, func2, type) \
-  type rhsV0; \
-  MICRO_MMA_TYPE_PEEL(func,func2,type,0);
-
-#define MICRO_MMA_ONE_PEEL \
-  if (sizeof(Scalar) == sizeof(float)) { \
-    MICRO_MMA_UNROLL_TYPE_PEEL(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, RhsPacket); \
-  } else { \
-    MICRO_MMA_UNROLL_TYPE_PEEL(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, __vector_pair); \
-  } \
-  rhs_ptr += (accRows * PEEL_MMA);
-
-#define MICRO_MMA_ONE \
-  if (sizeof(Scalar) == sizeof(float)) { \
-    MICRO_MMA_UNROLL_TYPE_ONE(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, RhsPacket); \
-  } else { \
-    MICRO_MMA_UNROLL_TYPE_ONE(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, __vector_pair); \
-  } \
-  rhs_ptr += accRows;
-
-#define MICRO_MMA_DST_PTR_ONE(iter) \
-  if (unroll_factor > iter) { \
-    bsetzeroMMA<Scalar, Packet>(&accZero##iter); \
-  } else { \
-    EIGEN_UNUSED_VARIABLE(accZero##iter); \
+#define MICRO_MMA_LOAD1_TWO(lhs_ptr, left)                                                        \
+  if (unroll_factor > left) {                                                                     \
+    if (MICRO_NORMAL(left)) {                                                                     \
+      ploadLhsMMA(reinterpret_cast<const double*>(lhs_ptr##left), plhsV##left);                   \
+      __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&lhsV2##left.packet), &plhsV##left); \
+      lhs_ptr##left += accCols * 2;                                                               \
+    } else {                                                                                      \
+      lhsV2##left.packet[0] = ploadLhs<Packet>(lhs_ptr##left);                                    \
+      lhsV2##left.packet[1] = ploadLhs<Packet>(lhs_ptr##left + accCols2);                         \
+      lhs_ptr##left += accCols2 * 2;                                                              \
+      EIGEN_UNUSED_VARIABLE(plhsV##left);                                                         \
+    }                                                                                             \
+  } else {                                                                                        \
+    EIGEN_UNUSED_VARIABLE(lhsV2##left);                                                           \
+    EIGEN_UNUSED_VARIABLE(plhsV##left);                                                           \
   }
 
-#define MICRO_MMA_DST_PTR MICRO_MMA_UNROLL(MICRO_MMA_DST_PTR_ONE)
+#define MICRO_MMA_LOAD_TWO(left) MICRO_MMA_LOAD1_TWO(lhs_ptr, left)
+#endif
+
+#define MICRO_MMA_UNROLL_ITER(func, val)                       \
+  func(val, 0) if (accItr > 1) {                               \
+    func(val, 1) if (accItr > 2) { func(val, 2) func(val, 3) } \
+  }
+
+#define MICRO_MMA_LOAD_ONE_RHS1(peel, right) ploadRhsMMA(rhs_ptr##right + (accRows * peel), rhsV##right[peel]);
+
+#define MICRO_MMA_LOAD_ONE_RHS(peel) MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_ONE_RHS1, peel)
+
+#define MICRO_MMA_TYPE_PEEL(funcw, funcl, type, peel)              \
+  if (PEEL_MMA > peel) {                                           \
+    Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \
+    MICRO_MMA_LOAD_ONE_RHS(peel)                                   \
+    MICRO_MMA_UNROLL(funcl)                                        \
+    MICRO_MMA_WORK(funcw, type, peel)                              \
+  }
 
-#define MICRO_MMA_SRC_PTR_ONE(iter) \
-  if (unroll_factor > iter) { \
-    lhs_ptr##iter = lhs_base + ( (row/accCols) + iter )*strideA*accCols + accCols*offsetA; \
-  } else { \
-    EIGEN_UNUSED_VARIABLE(lhs_ptr##iter); \
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type)                                                  \
+  type rhsV0[8], rhsV1[(accItr > 1) ? 8 : 1], rhsV2[(accItr > 2) ? 8 : 1], rhsV3[(accItr > 2) ? 8 : 1]; \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 0)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 1)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 2)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 3)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 4)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 5)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 6) MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 7)
+#else
+#define MICRO_MMA_LOAD_TWO_RHS(peel1, right)                                                      \
+  ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr##right + (accRows * peel1)), prhsV##peel1); \
+  __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsV##right[peel1]), &prhsV##peel1);
+
+#define MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, peel1, peel2)           \
+  if (PEEL_MMA > peel2) {                                                                  \
+    PacketBlock<Packet, 2> lhsV20, lhsV21, lhsV22, lhsV23, lhsV24, lhsV25, lhsV26, lhsV27; \
+    __vector_pair plhsV0, plhsV1, plhsV2, plhsV3, plhsV4, plhsV5, plhsV6, plhsV7;          \
+    if (sizeof(type) == 16) {                                                              \
+      MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_TWO_RHS, peel1)                                 \
+    } else {                                                                               \
+      EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                                 \
+      MICRO_MMA_LOAD_ONE_RHS(peel1)                                                        \
+      MICRO_MMA_LOAD_ONE_RHS(peel2)                                                        \
+    }                                                                                      \
+    MICRO_MMA_UNROLL(funcl2)                                                               \
+    MICRO_MMA_WORK(funcw2, type, peel1)                                                    \
+    MICRO_MMA_WORK(funcw2, type, peel2)                                                    \
+  } else {                                                                                 \
+    EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                                   \
+    MICRO_MMA_TYPE_PEEL(funcw1, funcl1, type, peel1)                                       \
   }
 
-#define MICRO_MMA_SRC_PTR MICRO_MMA_UNROLL(MICRO_MMA_SRC_PTR_ONE)
+#define MICRO_MMA_UNROLL_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type)                               \
+  type rhsV0[8], rhsV1[(accItr > 1) ? 8 : 1], rhsV2[(accItr > 2) ? 8 : 1], rhsV3[(accItr > 2) ? 8 : 1]; \
+  __vector_pair prhsV0, prhsV2, prhsV4, prhsV6;                                                         \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 0, 1)                                      \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 2, 3)                                      \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 4, 5)                                      \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 6, 7)
+#endif
+
+#define MICRO_MMA_UNROLL_TYPE_ONE(funcw, funcl, type) \
+  type rhsV0[1], rhsV1[1], rhsV2[1], rhsV3[1];        \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 0)
+
+#define MICRO_MMA_UPDATE_RHS1(size, right) rhs_ptr##right += (accRows * size);
+
+#define MICRO_MMA_UPDATE_RHS(size) MICRO_MMA_UNROLL_ITER(MICRO_MMA_UPDATE_RHS1, size)
+
+#define MICRO_MMA_UNROLL_TYPE(MICRO_MMA_TYPE, size)             \
+  MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, RhsPacket) \
+  MICRO_MMA_UPDATE_RHS(size)
+
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_MMA_ONE_PEEL MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_PEEL, PEEL_MMA)
+#else
+#define MICRO_MMA_UNROLL_TYPE2(MICRO_MMA_TYPE, size)                                                    \
+  MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, MICRO_MMA_WORK_TWO, MICRO_MMA_LOAD_TWO, RhsPacket) \
+  MICRO_MMA_UPDATE_RHS(size)
+
+#define MICRO_MMA_ONE_PEEL MICRO_MMA_UNROLL_TYPE2(MICRO_MMA_UNROLL_TYPE_PEEL2, PEEL_MMA)
+#endif
+
+#define MICRO_MMA_ONE MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_ONE, 1)
 
-#define MICRO_MMA_PREFETCH_ONE(iter) \
-  if (unroll_factor > iter) { \
-    EIGEN_POWER_PREFETCH(lhs_ptr##iter); \
+#define MICRO_MMA_DST_PTR_ONE(iter)       \
+  if (unroll_factor * accItr > iter) {    \
+    bsetzeroMMA(&accZero##iter);          \
+  } else {                                \
+    EIGEN_UNUSED_VARIABLE(accZero##iter); \
   }
 
-#define MICRO_MMA_PREFETCH MICRO_MMA_UNROLL(MICRO_MMA_PREFETCH_ONE)
+#define MICRO_MMA_DST_PTR MICRO_MMA_UNROLL(MICRO_MMA_DST_PTR_ONE)
+
+#define MICRO_MMA_SRC_PTR MICRO_MMA_UNROLL(MICRO_SRC_PTR_ONE)
+
+#define MICRO_MMA_PREFETCH MICRO_MMA_UNROLL(MICRO_PREFETCH_ONE)
 
-#define MICRO_MMA_STORE_ONE(iter) \
-  if (unroll_factor > iter) { \
-    storeAccumulator<DataMapper, Index, Packet, accCols>(row + iter*accCols, col, res, pAlpha, &accZero##iter); \
+#define MICRO_MMA_STORE_ONE(iter, left, right)                                                                 \
+  if (unroll_factor > left) {                                                                                  \
+    storeAccumulator<DataMapper, Packet, MICRO_NORMAL_PARTIAL(left)>(row + left * accCols, res##right, pAlpha, \
+                                                                     accCols2, &accZero##iter);                \
   }
 
-#define MICRO_MMA_STORE MICRO_MMA_UNROLL(MICRO_MMA_STORE_ONE)
+#define MICRO_MMA_ITER_UNROLL(func)                                                                                 \
+  if (accItr == 1) {                                                                                                \
+    func(0, 0, 0) func(1, 1, 0) func(2, 2, 0) func(3, 3, 0) func(4, 4, 0) func(5, 5, 0) func(6, 6, 0) func(7, 7, 0) \
+  } else if (accItr == 2) {                                                                                         \
+    func(0, 0, 0) func(1, 0, 1) func(2, 1, 0) func(3, 1, 1) func(4, 2, 0) func(5, 2, 1) func(6, 3, 0) func(7, 3, 1) \
+  } else {                                                                                                          \
+    func(0, 0, 0) func(1, 0, 1) func(2, 0, 2) func(3, 0, 3) func(4, 1, 0) func(5, 1, 1) func(6, 1, 2) func(7, 1, 3) \
+  }
 
-template<int unroll_factor, typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, typename Index, const Index accRows, const Index accCols>
-EIGEN_STRONG_INLINE void gemm_unrolled_MMA_iteration(
-  const DataMapper& res,
-  const Scalar* lhs_base,
-  const Scalar* rhs_base,
-  Index depth,
-  Index strideA,
-  Index offsetA,
-  Index& row,
-  Index col,
-  const Packet& pAlpha)
-{
-  const Scalar* rhs_ptr = rhs_base;
-  const Scalar* lhs_ptr0 = NULL, * lhs_ptr1 = NULL, * lhs_ptr2 = NULL, * lhs_ptr3 = NULL, * lhs_ptr4 = NULL, * lhs_ptr5 = NULL, * lhs_ptr6 = NULL, * lhs_ptr7 = NULL;
+#define MICRO_MMA_STORE MICRO_MMA_ITER_UNROLL(MICRO_MMA_STORE_ONE)
+
+#define MICRO_MMA_EXTRA_ROWS(right)                                                                           \
+  gemm_extra_row<Scalar, Packet, DataMapper, accRows, accCols>(                                               \
+      res3##right, blockA, rhs_base + right * accRows * strideB, depth, strideA, offsetA, strideB, row, rows, \
+      remaining_rows, pAlpha, pMask);
+
+#define MICRO_MMA_EXTRA_ROWS1(val, right) MICRO_MMA_EXTRA_ROWS(right);
+
+template <int unroll_factor, typename Scalar, typename Packet, typename RhsPacket, typename DataMapper,
+          const Index accRows, const Index accCols, bool full, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemm_unrolled_MMA_iteration(const DataMapper& res0, const DataMapper& res1,
+                                                     const DataMapper& res2, const DataMapper& res3,
+                                                     const Scalar* lhs_base, const Scalar* rhs_base, Index depth,
+                                                     Index strideA, Index strideB, Index offsetA, Index& row,
+                                                     const Packet& pAlpha, Index accCols2) {
+  const Scalar *rhs_ptr0 = rhs_base, *rhs_ptr1 = NULL, *rhs_ptr2 = NULL, *rhs_ptr3 = NULL;
+  const Scalar *lhs_ptr0 = NULL, *lhs_ptr1 = NULL, *lhs_ptr2 = NULL, *lhs_ptr3 = NULL, *lhs_ptr4 = NULL,
+               *lhs_ptr5 = NULL, *lhs_ptr6 = NULL, *lhs_ptr7 = NULL;
   __vector_quad accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7;
 
+  if (accItr > 1) {
+    rhs_ptr1 = rhs_base + (accRows * strideB);
+  } else {
+    EIGEN_UNUSED_VARIABLE(strideB);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr1);
+    EIGEN_UNUSED_VARIABLE(res1);
+  }
+  if (accItr > 2) {
+    rhs_ptr2 = rhs_base + (2 * accRows * strideB);
+    rhs_ptr3 = rhs_base + (3 * accRows * strideB);
+  } else {
+    EIGEN_UNUSED_VARIABLE(rhs_ptr2);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr3);
+    EIGEN_UNUSED_VARIABLE(res2);
+    EIGEN_UNUSED_VARIABLE(res3);
+  }
+
   MICRO_MMA_SRC_PTR
   MICRO_MMA_DST_PTR
 
-  Index k = 0;
-  for(; k + PEEL_MMA <= depth; k+= PEEL_MMA)
-  {
+  Index k = 0, depth2 = depth - PEEL_MMA;
+  for (; k <= depth2; k += PEEL_MMA) {
     EIGEN_POWER_PREFETCH(rhs_ptr);
     MICRO_MMA_PREFETCH
     MICRO_MMA_ONE_PEEL
   }
-  for(; k < depth; k++)
-  {
+  for (; k < depth; k++) {
     MICRO_MMA_ONE
   }
   MICRO_MMA_STORE
 
-  row += unroll_factor*accCols;
+  MICRO_UPDATE
 }
 
-template<typename Scalar, typename Index, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols>
-void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
-{
-      const Index remaining_rows = rows % accCols;
-      const Index remaining_cols = cols % accRows;
+#define MICRO_MMA_UNROLL_ITER2(N, M)                                                                                 \
+  gemm_unrolled_MMA_iteration<N + (M ? 1 : 0), Scalar, Packet, RhsPacket, DataMapper, accRows, accCols, !M, accItr>( \
+      res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, strideB, offsetA, row, pAlpha,                 \
+      M ? remaining_rows : accCols);                                                                                 \
+  if (M) return;
 
-      if( strideA == -1 ) strideA = depth;
-      if( strideB == -1 ) strideB = depth;
+#define MICRO_MMA_ROWS(n)             \
+  while (row + n * accCols <= rows) { \
+    MICRO_MMA_UNROLL_ITER2(n, 0);     \
+  }
 
-      const Packet pAlpha = pset1<Packet>(alpha);
-      const Packet pMask  = bmask<Packet>((const int)(remaining_rows));
+template <typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows,
+          const Index accCols, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemmMMA_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index depth,
+                                      Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows,
+                                      Index remaining_rows, const Packet& pAlpha, const Packet& pMask) {
+  const DataMapper res30 = res.getSubMapper(0, col);
+  const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows * 1) : res30;
+  const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows * 2) : res30;
+  const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows * 3) : res30;
 
-      Index col = 0;
-      for(; col + accRows <= cols; col += accRows)
-      {
-        const Scalar* rhs_base = blockB + col*strideB + accRows*offsetB;
-        const Scalar* lhs_base = blockA;
+  const Scalar* rhs_base = blockB + col * strideB + accRows * offsetB;
+  const Scalar* lhs_base = blockA + accCols * offsetA;
+  Index row = 0;
 
-        Index row = 0;
 #define MAX_MMA_UNROLL 7
-        while(row + MAX_MMA_UNROLL*accCols <= rows) {
-          gemm_unrolled_MMA_iteration<MAX_MMA_UNROLL, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
-        }
-        switch( (rows-row)/accCols ) {
+
+#if MAX_MMA_UNROLL < 2
+  if (1) {
+#elif MAX_MMA_UNROLL < 4
+  if (accItr <= 2) {
+#else
+  if (accItr == 1) {
+#endif
+    MICRO_MMA_ROWS(MAX_MMA_UNROLL);
+  } else if (accItr == 2) {
+    MICRO_MMA_ROWS(4);
+  } else {
+    MICRO_MMA_ROWS(2);
+  }
+  switch ((rows - row) / accCols) {
 #if MAX_MMA_UNROLL > 7
-          case 7:
-            gemm_unrolled_MMA_iteration<7, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
-            break;
+    case 7:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 7)
+      }
+      break;
 #endif
 #if MAX_MMA_UNROLL > 6
-          case 6:
-            gemm_unrolled_MMA_iteration<6, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
-            break;
+    case 6:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 6)
+      }
+      break;
 #endif
 #if MAX_MMA_UNROLL > 5
-          case 5:
-            gemm_unrolled_MMA_iteration<5, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
-            break;
+    case 5:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 5)
+      }
+      break;
 #endif
 #if MAX_MMA_UNROLL > 4
-          case 4:
-            gemm_unrolled_MMA_iteration<4, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
-            break;
+    case 4:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 4)
+      }
+      break;
 #endif
 #if MAX_MMA_UNROLL > 3
-          case 3:
-            gemm_unrolled_MMA_iteration<3, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
-            break;
+    case 3:
+      if (accItr <= 2) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 3)
+      }
+      break;
 #endif
 #if MAX_MMA_UNROLL > 2
-          case 2:
-            gemm_unrolled_MMA_iteration<2, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
-            break;
+    case 2:
+      if (accItr <= 2) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 2)
+      }
+      break;
 #endif
 #if MAX_MMA_UNROLL > 1
-          case 1:
-            gemm_unrolled_MMA_iteration<1, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
-            break;
+    case 1:
+      MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 1)
+      break;
 #endif
-          default:
-            break;
-        }
+    default:
+      break;
+  }
 #undef MAX_MMA_UNROLL
 
-        if(remaining_rows > 0)
-        {
-          gemm_extra_row<Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, rows, cols, remaining_rows, pAlpha, pMask);
-        }
-      }
+  if (remaining_rows > 0) {
+    MICRO_MMA_UNROLL_ITER(MICRO_MMA_EXTRA_ROWS1, 0)
+  }
+}
 
-      if(remaining_cols > 0)
-      {
-        const Scalar* rhs_base = blockB + col*strideB + remaining_cols*offsetB;
-        const Scalar* lhs_base = blockA;
+#define MICRO_MMA_COLS(n)                                                                                          \
+  for (; col + n * accRows <= cols; col += n * accRows) {                                                          \
+    gemmMMA_cols<Scalar, Packet, RhsPacket2, DataMapper, accRows, accCols, n>(                                     \
+        res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlpha, pMask); \
+  }
 
-        for(; col < cols; col++)
-        {
-          Index row = 0;
+template <typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows,
+          const Index accCols>
+void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols,
+             Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
+  const Index remaining_rows = rows % accCols;
 
-          gemm_unrolled_col<Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, rows, col, remaining_cols, pAlpha);
+  if (strideA == -1) strideA = depth;
+  if (strideB == -1) strideB = depth;
 
-          if (remaining_rows > 0)
-          {
-            gemm_extra_col<Scalar, Packet, DataMapper, Index, accRows>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_rows, remaining_cols, pAlpha);
-          }
-          rhs_base++;
-        }
-      }
+  const Packet pAlpha = pset1<Packet>(alpha);
+  const Packet pMask = bmask<Packet>(remaining_rows);
+
+  typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2;
+
+  Index col = 0;
+#ifdef GEMM_MULTIPLE_COLS
+  MICRO_MMA_COLS(4);
+  MICRO_MMA_COLS(2);
+#endif
+  MICRO_MMA_COLS(1);
+
+  if (col != cols) {
+    gemm_extra_cols<Scalar, Packet, DataMapper, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB,
+                                                         col, rows, cols, remaining_rows, pAlpha, pMask);
+  }
 }
 
-#define accColsC (accCols / 2)
 #define advanceRows ((LhsIsReal) ? 1 : 2)
 #define advanceCols ((RhsIsReal) ? 1 : 2)
 
 // PEEL_COMPLEX_MMA loop factor.
-#define PEEL_COMPLEX_MMA 7
-
-#define MICRO_COMPLEX_MMA_UNROLL(func) \
-  func(0) func(1) func(2) func(3) func(4)
-
-#define MICRO_COMPLEX_MMA_LOAD_ONE(iter) \
-  if (unroll_factor > iter) { \
-    lhsV##iter = ploadLhs<Scalar, Packet>(lhs_ptr_real##iter); \
-    lhs_ptr_real##iter += accCols; \
-    if(!LhsIsReal) { \
-      lhsVi##iter = ploadLhs<Scalar, Packet>(lhs_ptr_imag##iter); \
-      lhs_ptr_imag##iter += accCols; \
-    } else { \
-      EIGEN_UNUSED_VARIABLE(lhsVi##iter); \
-    } \
-  } else { \
-    EIGEN_UNUSED_VARIABLE(lhsV##iter); \
-    EIGEN_UNUSED_VARIABLE(lhsVi##iter); \
-  }
-
-#define MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel) \
-  if (unroll_factor > iter) { \
-    pgercMMA<Scalar, Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \
-  }
-
-#define MICRO_COMPLEX_MMA_TYPE_PEEL(func, func2, type, peel) \
-  if (PEEL_COMPLEX_MMA > peel) { \
-    Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4; \
-    Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3, lhsVi4; \
-    ploadRhsMMA<Scalar, type>(rhs_ptr_real + (accRows * peel), rhsV##peel); \
-    if(!RhsIsReal) { \
-      ploadRhsMMA<Scalar, type>(rhs_ptr_imag + (accRows * peel), rhsVi##peel); \
-    } else { \
-      EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
-    } \
-    MICRO_COMPLEX_MMA_UNROLL(func2); \
-    func(0,type,peel) func(1,type,peel) func(2,type,peel) func(3,type,peel) func(4,type,peel) \
-  } else { \
-    EIGEN_UNUSED_VARIABLE(rhsV##peel); \
-    EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
-  }
-
-#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(func, func2, type) \
-  type rhsV0, rhsV1, rhsV2, rhsV3, rhsV4, rhsV5, rhsV6, rhsV7, rhsV8, rhsV9; \
-  type rhsVi0, rhsVi1, rhsVi2, rhsVi3, rhsVi4, rhsVi5, rhsVi6, rhsVi7, rhsVi8, rhsVi9; \
-  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,0); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,1); \
-  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,2); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,3); \
-  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,4); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,5); \
-  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,6); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,7); \
-  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,8); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,9);
-
-#define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(func, func2, type) \
-  type rhsV0, rhsVi0; \
-  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,0);
-
-#define MICRO_COMPLEX_MMA_ONE_PEEL \
-  if (sizeof(Scalar) == sizeof(float)) { \
-    MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, RhsPacket); \
-  } else { \
-    MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, __vector_pair); \
-  } \
-  rhs_ptr_real += (accRows * PEEL_COMPLEX_MMA); \
-  if(!RhsIsReal) rhs_ptr_imag += (accRows * PEEL_COMPLEX_MMA);
-
-#define MICRO_COMPLEX_MMA_ONE \
-  if (sizeof(Scalar) == sizeof(float)) { \
-    MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, RhsPacket); \
-  } else { \
-    MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, __vector_pair); \
-  } \
-  rhs_ptr_real += accRows; \
-  if(!RhsIsReal) rhs_ptr_imag += accRows;
+#ifdef GEMM_MULTIPLE_COLS
+#define PEEL_COMPLEX_MMA 4
+#else
+#define PEEL_COMPLEX_MMA 3
+#endif
+
+#define MICRO_COMPLEX_MMA_UNROLL(func) func(0) func(1) func(2) func(3)
+
+#define MICRO_COMPLEX_MMA_WORK(func, type, peel)                                                            \
+  if (accItr == 1) {                                                                                        \
+    func(0, type, peel, 0, 0) func(1, type, peel, 1, 0) func(2, type, peel, 2, 0) func(3, type, peel, 3, 0) \
+  } else if (accItr == 2) {                                                                                 \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 1, 0) func(3, type, peel, 1, 1) \
+  } else {                                                                                                  \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 0, 2) func(3, type, peel, 0, 3) \
+  }
+
+#define MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel, left, right)                                        \
+  if (unroll_factor > left) {                                                                            \
+    pgercMMA<Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(                            \
+        &accReal##iter, &accImag##iter, lhsV##left, lhsVi##left, rhsV##right[peel], rhsVi##right[peel]); \
+  }
+
+#ifdef VECTOR_PAIR_LOADS_LHS
+#define MICRO_COMPLEX_MMA_WORK_TWO(iter, type, peel, left, right)                                    \
+  if (unroll_factor > left) {                                                                        \
+    pgercMMA<Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(                        \
+        &accReal##iter, &accImag##iter, lhsV2##left.packet[peel & 1], lhsVi2##left.packet[peel & 1], \
+        rhsV##right[peel], rhsVi##right[peel]);                                                      \
+  }
+
+#define MICRO_COMPLEX_MMA_LOAD1_TWO(lhs_ptr, left)                                                  \
+  if (!LhsIsReal && (unroll_factor > left)) {                                                       \
+    if (MICRO_NORMAL(left)) {                                                                       \
+      ploadLhsMMA(reinterpret_cast<const double*>(lhs_ptr_real##left + imag_delta), plhsVi##left);  \
+      __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&lhsVi2##left.packet), &plhsVi##left); \
+    } else {                                                                                        \
+      lhsVi2##left.packet[0] = ploadLhs<Packet>(lhs_ptr_real##left + imag_delta2);                  \
+      lhsVi2##left.packet[1] = ploadLhs<Packet>(lhs_ptr_real##left + imag_delta2 + accCols2);       \
+      EIGEN_UNUSED_VARIABLE(plhsVi##left);                                                          \
+    }                                                                                               \
+  } else {                                                                                          \
+    EIGEN_UNUSED_VARIABLE(lhsVi2##left);                                                            \
+    EIGEN_UNUSED_VARIABLE(plhsVi##left);                                                            \
+  }                                                                                                 \
+  MICRO_MMA_LOAD1_TWO(lhs_ptr_real, left)
+
+#define MICRO_COMPLEX_MMA_LOAD_TWO(left) MICRO_COMPLEX_MMA_LOAD1_TWO(lhs_ptr, left)
+#endif
+
+#define MICRO_COMPLEX_MMA_LOAD_RHS1(peel, right)                             \
+  ploadRhsMMA(rhs_ptr_real##right + (accRows * peel), rhsV##right[peel]);    \
+  if (!RhsIsReal) {                                                          \
+    ploadRhsMMA(rhs_ptr_imag##right + (accRows * peel), rhsVi##right[peel]); \
+  }
+
+#define MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel) MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_RHS1, peel)
+
+#define MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, peel) \
+  if (PEEL_COMPLEX_MMA > peel) {                              \
+    Packet lhsV0, lhsV1, lhsV2, lhsV3;                        \
+    Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3;                    \
+    MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel)                      \
+    MICRO_COMPLEX_MMA_UNROLL(funcl)                           \
+    MICRO_COMPLEX_MMA_WORK(funcw, type, peel)                 \
+  }
+
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type)                                                      \
+  type rhsV0[4], rhsVi0[4], rhsV1[(accItr > 1) ? 4 : 1], rhsVi1[(accItr > 1) ? 4 : 1], rhsV2[(accItr > 2) ? 4 : 1], \
+      rhsVi2[(accItr > 2) ? 4 : 1], rhsV3[(accItr > 2) ? 4 : 1], rhsVi3[(accItr > 2) ? 4 : 1];                      \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 0)                                                                \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 1)                                                                \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 2) MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 3)
+#else
+#define MICRO_COMPLEX_MMA_LOAD_TWO_RHS(peel1, right)                                                      \
+  ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr_real##right + (accRows * peel1)), prhsV##peel1);    \
+  __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsV##right[peel1]), &prhsV##peel1);            \
+  if (!RhsIsReal) {                                                                                       \
+    ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr_imag##right + (accRows * peel1)), prhsVi##peel1); \
+    __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsVi##right[peel1]), &prhsVi##peel1);        \
+  } else {                                                                                                \
+    EIGEN_UNUSED_VARIABLE(prhsVi##peel1);                                                                 \
+  }
+
+#define MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, peel1, peel2) \
+  if (PEEL_COMPLEX_MMA > peel2) {                                                        \
+    PacketBlock<Packet, 2> lhsV20, lhsV21, lhsV22, lhsV23;                               \
+    PacketBlock<Packet, 2> lhsVi20, lhsVi21, lhsVi22, lhsVi23;                           \
+    __vector_pair plhsV0, plhsV1, plhsV2, plhsV3;                                        \
+    __vector_pair plhsVi0, plhsVi1, plhsVi2, plhsVi3;                                    \
+    if (sizeof(type) == 16) {                                                            \
+      MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_TWO_RHS, peel1)                       \
+    } else {                                                                             \
+      EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                               \
+      EIGEN_UNUSED_VARIABLE(prhsVi##peel1);                                              \
+      MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel1);                                             \
+      MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel2);                                             \
+    }                                                                                    \
+    MICRO_COMPLEX_MMA_UNROLL(funcl2)                                                     \
+    MICRO_COMPLEX_MMA_WORK(funcw2, type, peel1)                                          \
+    MICRO_COMPLEX_MMA_WORK(funcw2, type, peel2)                                          \
+  } else {                                                                               \
+    EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                                 \
+    EIGEN_UNUSED_VARIABLE(prhsVi##peel1);                                                \
+    MICRO_COMPLEX_MMA_TYPE_PEEL(funcw1, funcl1, type, peel1)                             \
+  }
+
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type)                                   \
+  type rhsV0[4], rhsVi0[4], rhsV1[(accItr > 1) ? 4 : 1], rhsVi1[(accItr > 1) ? 4 : 1], rhsV2[(accItr > 2) ? 4 : 1], \
+      rhsVi2[(accItr > 2) ? 4 : 1], rhsV3[(accItr > 2) ? 4 : 1], rhsVi3[(accItr > 2) ? 4 : 1];                      \
+  __vector_pair prhsV0, prhsV2;                                                                                     \
+  __vector_pair prhsVi0, prhsVi2;                                                                                   \
+  MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 0, 1)                                          \
+  MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 2, 3)
+#endif
+
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(funcw, funcl, type)                              \
+  type rhsV0[1], rhsVi0[1], rhsV1[1], rhsVi1[1], rhsV2[1], rhsVi2[1], rhsV3[1], rhsVi3[1]; \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 0)
+
+#define MICRO_COMPLEX_MMA_UPDATE_RHS1(size, right) \
+  rhs_ptr_real##right += (accRows * size);         \
+  if (!RhsIsReal) rhs_ptr_imag##right += (accRows * size);
+
+#define MICRO_COMPLEX_MMA_UPDATE_RHS(size) MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_UPDATE_RHS1, size)
+
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_TYPE, size)                     \
+  MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, RhsPacket) \
+  MICRO_COMPLEX_MMA_UPDATE_RHS(size);
+
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_COMPLEX_MMA_ONE_PEEL MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL, PEEL_COMPLEX_MMA)
+#else
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE2(MICRO_COMPLEX_MMA_TYPE, size)                                     \
+  MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, MICRO_COMPLEX_MMA_WORK_TWO, \
+                         MICRO_COMPLEX_MMA_LOAD_TWO, RhsPacket)                                          \
+  MICRO_COMPLEX_MMA_UPDATE_RHS(size);
+
+#define MICRO_COMPLEX_MMA_ONE_PEEL MICRO_COMPLEX_MMA_UNROLL_TYPE2(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL2, PEEL_COMPLEX_MMA)
+#endif
+
+#define MICRO_COMPLEX_MMA_ONE MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE, 1)
 
 #define MICRO_COMPLEX_MMA_DST_PTR_ONE(iter) \
-  if (unroll_factor > iter) { \
-    bsetzeroMMA<Scalar, Packet>(&accReal##iter); \
-    bsetzeroMMA<Scalar, Packet>(&accImag##iter); \
-  } else { \
-    EIGEN_UNUSED_VARIABLE(accReal##iter); \
-    EIGEN_UNUSED_VARIABLE(accImag##iter); \
+  if (unroll_factor * accItr > iter) {      \
+    bsetzeroMMA(&accReal##iter);            \
+    bsetzeroMMA(&accImag##iter);            \
+  } else {                                  \
+    EIGEN_UNUSED_VARIABLE(accReal##iter);   \
+    EIGEN_UNUSED_VARIABLE(accImag##iter);   \
   }
 
 #define MICRO_COMPLEX_MMA_DST_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_DST_PTR_ONE)
 
-#define MICRO_COMPLEX_MMA_SRC_PTR_ONE(iter) \
-  if (unroll_factor > iter) { \
-    lhs_ptr_real##iter = lhs_base + ( ((advanceRows*row)/accCols) + iter*advanceRows )*strideA*accCols + accCols*offsetA; \
-    if(!LhsIsReal) { \
-      lhs_ptr_imag##iter = lhs_ptr_real##iter + accCols*strideA; \
-    } else { \
-      EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \
-    } \
-  } else { \
-    EIGEN_UNUSED_VARIABLE(lhs_ptr_real##iter); \
-    EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \
-  }
-
-#define MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_SRC_PTR_ONE)
-
-#define MICRO_COMPLEX_MMA_PREFETCH_ONE(iter) \
-  if (unroll_factor > iter) { \
-    EIGEN_POWER_PREFETCH(lhs_ptr_real##iter); \
-    if(!LhsIsReal) { \
-      EIGEN_POWER_PREFETCH(lhs_ptr_imag##iter); \
-    } \
-  }
-
-#define MICRO_COMPLEX_MMA_PREFETCH MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_PREFETCH_ONE)
-
-#define MICRO_COMPLEX_MMA_STORE_ONE(iter) \
-  if (unroll_factor > iter) { \
-    storeComplexAccumulator<DataMapper, Index, Packet, Packetc, accColsC, 0>(row + iter*accCols, col, res, pAlphaReal, pAlphaImag, &accReal##iter, &accImag##iter); \
-  }
-
-#define MICRO_COMPLEX_MMA_STORE MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE)
-
-template<int unroll_factor, typename Scalar, typename Packet, typename Packetc, typename RhsPacket, typename DataMapper, typename Index, const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
-EIGEN_STRONG_INLINE void gemm_complex_unrolled_MMA_iteration(
-  const DataMapper& res,
-  const Scalar* lhs_base,
-  const Scalar* rhs_base,
-  Index depth,
-  Index strideA,
-  Index offsetA,
-  Index strideB,
-  Index& row,
-  Index col,
-  const Packet& pAlphaReal,
-  const Packet& pAlphaImag)
-{
-  const Scalar* rhs_ptr_real = rhs_base;
-  const Scalar* rhs_ptr_imag;
-  if(!RhsIsReal) {
-    rhs_ptr_imag = rhs_base + accRows*strideB;
+#define MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE)
+
+#define MICRO_COMPLEX_MMA_PREFETCH MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_PREFETCH_ONE)
+
+#define MICRO_COMPLEX_MMA_STORE_ONE(iter, left, right)                                                                 \
+  if (unroll_factor > left) {                                                                                          \
+    storeComplexAccumulator<DataMapper, Packet, Packetc, accCols, (unroll_factor != (left + 1)) ? accCols : accCols2>( \
+        row + left * accCols, res##right, pAlphaReal, pAlphaImag, pMask, &accReal##iter, &accImag##iter);              \
+  }
+
+#define MICRO_COMPLEX_MMA_ITER_UNROLL(func)                 \
+  if (accItr == 1) {                                        \
+    func(0, 0, 0) func(1, 1, 0) func(2, 2, 0) func(3, 3, 0) \
+  } else if (accItr == 2) {                                 \
+    func(0, 0, 0) func(1, 0, 1) func(2, 1, 0) func(3, 1, 1) \
+  } else {                                                  \
+    func(0, 0, 0) func(1, 0, 1) func(2, 0, 2) func(3, 0, 3) \
+  }
+
+#define MICRO_COMPLEX_MMA_STORE MICRO_COMPLEX_MMA_ITER_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE)
+
+#define MICRO_COMPLEX_MMA_EXTRA_ROWS(right)                                                                            \
+  gemm_complex_extra_row<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, \
+                         RhsIsReal>(res3##right, blockA, rhs_base + right * accRows * (RhsIsReal ? 1 : 2) * strideB,   \
+                                    depth, strideA, offsetA, strideB, row, rows, remaining_rows, pAlphaReal,           \
+                                    pAlphaImag, pMask);
+
+#define MICRO_COMPLEX_MMA_EXTRA_ROWS1(val, right) MICRO_COMPLEX_MMA_EXTRA_ROWS(right);
+
+template <int unroll_factor, typename Scalar, typename Packet, typename Packetc, typename RhsPacket,
+          typename DataMapper, const Index accRows, const Index accCols, const Index accCols2, bool ConjugateLhs,
+          bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemm_complex_unrolled_MMA_iteration(const DataMapper& res0, const DataMapper& res1,
+                                                             const DataMapper& res2, const DataMapper& res3,
+                                                             const Scalar* lhs_base, const Scalar* rhs_base,
+                                                             Index depth, Index strideA, Index offsetA, Index strideB,
+                                                             Index& row, const Packet& pAlphaReal,
+                                                             const Packet& pAlphaImag, const Packet& pMask) {
+  const Scalar *rhs_ptr_real0 = rhs_base, *rhs_ptr_real1 = NULL, *rhs_ptr_real2 = NULL, *rhs_ptr_real3 = NULL;
+  const Scalar *rhs_ptr_imag0 = NULL, *rhs_ptr_imag1 = NULL, *rhs_ptr_imag2 = NULL, *rhs_ptr_imag3 = NULL;
+  const Index imag_delta = accCols * strideA;
+  const Index imag_delta2 = accCols2 * strideA;
+
+  if (!RhsIsReal) {
+    rhs_ptr_imag0 = rhs_base + accRows * strideB;
   } else {
-    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag0);
   }
-  const Scalar* lhs_ptr_real0 = NULL, * lhs_ptr_imag0 = NULL, * lhs_ptr_real1 = NULL, * lhs_ptr_imag1 = NULL;
-  const Scalar* lhs_ptr_real2 = NULL, * lhs_ptr_imag2 = NULL, * lhs_ptr_real3 = NULL, * lhs_ptr_imag3 = NULL;
-  const Scalar* lhs_ptr_real4 = NULL, * lhs_ptr_imag4 = NULL;
-  __vector_quad accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3, accReal4, accImag4;
+  if (accItr > 1) {
+    if (!RhsIsReal) {
+      rhs_ptr_real1 = rhs_base + (2 * accRows * strideB);
+      rhs_ptr_imag1 = rhs_base + (3 * accRows * strideB);
+    } else {
+      rhs_ptr_real1 = rhs_base + accRows * strideB;
+      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1);
+    }
+  } else {
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_real1);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1);
+    EIGEN_UNUSED_VARIABLE(res1);
+  }
+  if (accItr > 2) {
+    if (!RhsIsReal) {
+      rhs_ptr_real2 = rhs_base + (4 * accRows * strideB);
+      rhs_ptr_imag2 = rhs_base + (5 * accRows * strideB);
+      rhs_ptr_real3 = rhs_base + (6 * accRows * strideB);
+      rhs_ptr_imag3 = rhs_base + (7 * accRows * strideB);
+    } else {
+      rhs_ptr_real2 = rhs_base + (2 * accRows * strideB);
+      rhs_ptr_real3 = rhs_base + (3 * accRows * strideB);
+      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2);
+      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3);
+    }
+  } else {
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_real2);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_real3);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3);
+    EIGEN_UNUSED_VARIABLE(res2);
+    EIGEN_UNUSED_VARIABLE(res3);
+  }
+  const Scalar *lhs_ptr_real0 = NULL, *lhs_ptr_real1 = NULL;
+  const Scalar *lhs_ptr_real2 = NULL, *lhs_ptr_real3 = NULL;
+  __vector_quad accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3;
 
   MICRO_COMPLEX_MMA_SRC_PTR
   MICRO_COMPLEX_MMA_DST_PTR
 
-  Index k = 0;
-  for(; k + PEEL_COMPLEX_MMA <= depth; k+= PEEL_COMPLEX_MMA)
-  {
+  Index k = 0, depth2 = depth - PEEL_COMPLEX_MMA;
+  for (; k <= depth2; k += PEEL_COMPLEX_MMA) {
     EIGEN_POWER_PREFETCH(rhs_ptr_real);
-    if(!RhsIsReal) {
+    if (!RhsIsReal) {
       EIGEN_POWER_PREFETCH(rhs_ptr_imag);
     }
     MICRO_COMPLEX_MMA_PREFETCH
     MICRO_COMPLEX_MMA_ONE_PEEL
   }
-  for(; k < depth; k++)
-  {
+  for (; k < depth; k++) {
     MICRO_COMPLEX_MMA_ONE
   }
   MICRO_COMPLEX_MMA_STORE
 
-  row += unroll_factor*accCols;
+  MICRO_COMPLEX_UPDATE
 }
 
-template<typename LhsScalar, typename RhsScalar, typename Scalarc, typename Scalar, typename Index, typename Packet, typename Packetc, typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
-void gemm_complexMMA(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
-{
-      const Index remaining_rows = rows % accCols;
-      const Index remaining_cols = cols % accRows;
+#define MICRO_COMPLEX_MMA_UNROLL_ITER2(N, M)                                                                           \
+  gemm_complex_unrolled_MMA_iteration<N + (M ? 1 : 0), Scalar, Packet, Packetc, RhsPacket, DataMapper, accRows,        \
+                                      accCols, M ? M : accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal,      \
+                                      accItr>(res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, offsetA, \
+                                              strideB, row, pAlphaReal, pAlphaImag, pMask);                            \
+  if (M) return;
 
-      if( strideA == -1 ) strideA = depth;
-      if( strideB == -1 ) strideB = depth;
-
-      const Packet pAlphaReal = pset1<Packet>(alpha.real());
-      const Packet pAlphaImag = pset1<Packet>(alpha.imag());
-      const Packet pMask = bmask<Packet>((const int)(remaining_rows));
-
-      const Scalar* blockA = (Scalar *) blockAc;
-      const Scalar* blockB = (Scalar *) blockBc;
+#define MICRO_COMPLEX_MMA_ROWS(n)         \
+  while (row + n * accCols <= rows) {     \
+    MICRO_COMPLEX_MMA_UNROLL_ITER2(n, 0); \
+  }
 
-      Index col = 0;
-      for(; col + accRows <= cols; col += accRows)
-      {
-        const Scalar* rhs_base = blockB + advanceCols*col*strideB + accRows*offsetB;
-        const Scalar* lhs_base = blockA;
-        Index row = 0;
+template <typename Scalar, typename Packet, typename Packetc, typename RhsPacket, typename DataMapper,
+          const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal,
+          bool RhsIsReal, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemmMMA_complex_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB,
+                                              Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB,
+                                              Index col, Index rows, Index remaining_rows, const Packet& pAlphaReal,
+                                              const Packet& pAlphaImag, const Packet& pMask) {
+  const DataMapper res30 = res.getSubMapper(0, col);
+  const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows * 1) : res30;
+  const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows * 2) : res30;
+  const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows * 3) : res30;
+
+  const Scalar* rhs_base = blockB + advanceCols * col * strideB + accRows * offsetB;
+  const Scalar* lhs_base = blockA + accCols * offsetA;
+  Index row = 0;
 
 #define MAX_COMPLEX_MMA_UNROLL 4
-        while(row + MAX_COMPLEX_MMA_UNROLL*accCols <= rows) {
-          gemm_complex_unrolled_MMA_iteration<MAX_COMPLEX_MMA_UNROLL, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
-        }
-        switch( (rows-row)/accCols ) {
-#if MAX_COMPLEX_MMA_UNROLL > 4
-          case 4:
-            gemm_complex_unrolled_MMA_iteration<4, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
-            break;
+
+#if MAX_COMPLEX_MMA_UNROLL < 2
+  if (1) {
+#elif MAX_COMPLEX_MMA_UNROLL < 4
+  if (accItr <= 2) {
+#else
+  if (accItr == 1) {
 #endif
+    MICRO_COMPLEX_MMA_ROWS(MAX_COMPLEX_MMA_UNROLL);
+  } else if (accItr == 2) {
+    MICRO_COMPLEX_MMA_ROWS(2);
+  } else {
+    MICRO_COMPLEX_MMA_ROWS(1);
+  }
+  switch ((rows - row) / accCols) {
 #if MAX_COMPLEX_MMA_UNROLL > 3
-          case 3:
-            gemm_complex_unrolled_MMA_iteration<3, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
-            break;
+    case 3:
+      if (accItr == 1) {
+        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 3)
+      }
+      break;
 #endif
 #if MAX_COMPLEX_MMA_UNROLL > 2
-          case 2:
-            gemm_complex_unrolled_MMA_iteration<2, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
-            break;
+    case 2:
+      if (accItr == 1) {
+        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 2)
+      }
+      break;
 #endif
 #if MAX_COMPLEX_MMA_UNROLL > 1
-          case 1:
-            gemm_complex_unrolled_MMA_iteration<1, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
-            break;
+    case 1:
+      if (accItr <= 2) {
+        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 1)
+      }
+      break;
 #endif
-          default:
-            break;
-        }
+    default:
+      break;
+  }
 #undef MAX_COMPLEX_MMA_UNROLL
 
-        if(remaining_rows > 0)
-        {
-          gemm_complex_extra_row<Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask);
-        }
-      }
+  if (remaining_rows > 0) {
+    MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_EXTRA_ROWS1, 0)
+  }
+}
 
-      if(remaining_cols > 0)
-      {
-        const Scalar* rhs_base = blockB + advanceCols*col*strideB + remaining_cols*offsetB;
-        const Scalar* lhs_base = blockA;
+#define MICRO_COMPLEX_MMA_COLS(n)                                                                                      \
+  for (; col + n * accRows <= cols; col += n * accRows) {                                                              \
+    gemmMMA_complex_cols<Scalar, Packet, Packetc, RhsPacket2, DataMapper, accRows, accCols, ConjugateLhs,              \
+                         ConjugateRhs, LhsIsReal, RhsIsReal, n>(res, blockA, blockB, depth, strideA, offsetA, strideB, \
+                                                                offsetB, col, rows, remaining_rows, pAlphaReal,        \
+                                                                pAlphaImag, pMask);                                    \
+  }
 
-        for(; col < cols; col++)
-        {
-          Index row = 0;
+template <typename LhsScalar, typename RhsScalar, typename Scalarc, typename Scalar, typename Packet, typename Packetc,
+          typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols, bool ConjugateLhs,
+          bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
+void gemm_complexMMA(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc, Index rows, Index depth,
+                     Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
+  const Index remaining_rows = rows % accCols;
 
-          gemm_complex_unrolled_col<Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, col, remaining_cols, pAlphaReal, pAlphaImag);
+  if (strideA == -1) strideA = depth;
+  if (strideB == -1) strideB = depth;
 
-          if (remaining_rows > 0)
-          {
-            gemm_complex_extra_col<Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_rows, remaining_cols, pAlphaReal, pAlphaImag);
-          }
-          rhs_base++;
-        }
-      }
+  const Packet pAlphaReal = pset1<Packet>(alpha.real());
+  const Packet pAlphaImag = pset1<Packet>(alpha.imag());
+  const Packet pMask = bmask<Packet>(remaining_rows);
+
+  const Scalar* blockA = (Scalar*)blockAc;
+  const Scalar* blockB = (Scalar*)blockBc;
+
+  typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2;
+
+  Index col = 0;
+#ifdef GEMM_MULTIPLE_COLS
+  MICRO_COMPLEX_MMA_COLS(4);
+  MICRO_COMPLEX_MMA_COLS(2);
+#endif
+  MICRO_COMPLEX_MMA_COLS(1);
+
+  if (col != cols) {
+    gemm_complex_extra_cols<Scalar, Packet, Packetc, DataMapper, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
+                            RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols,
+                                       remaining_rows, pAlphaReal, pAlphaImag, pMask);
+  }
 }
 
 #undef accColsC
 #undef advanceRows
 #undef advanceCols
 
-#pragma GCC reset_options
-} // end namespace internal
+}  // end namespace internal
 
-} // end namespace Eigen
+}  // end namespace Eigen
 
-#endif // EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
+#if defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH)
+#pragma GCC pop_options
+#endif
 
+#endif  // EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H