@smake/eigen 1.1.0 → 1.1.1

package/eigen/Eigen/src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h

@@ -33,37 +33,37 @@
 #ifndef EIGEN_GENERAL_MATRIX_MATRIX_TRIANGULAR_BLAS_H
 #define EIGEN_GENERAL_MATRIX_MATRIX_TRIANGULAR_BLAS_H
 
+// IWYU pragma: private
+#include "../InternalHeaderCheck.h"
+
 namespace Eigen {
 
 namespace internal {
 
 template <typename Index, typename Scalar, int AStorageOrder, bool ConjugateA, int ResStorageOrder, int UpLo>
-struct general_matrix_matrix_rankupdate :
-       general_matrix_matrix_triangular_product<
-         Index,Scalar,AStorageOrder,ConjugateA,Scalar,AStorageOrder,ConjugateA,ResStorageOrder,1,UpLo,BuiltIn> {};
-
+struct general_matrix_matrix_rankupdate
+    : general_matrix_matrix_triangular_product<Index, Scalar, AStorageOrder, ConjugateA, Scalar, AStorageOrder,
+                                               ConjugateA, ResStorageOrder, 1, UpLo, BuiltIn> {};
 
 // try to go to BLAS specialization
-#define EIGEN_BLAS_RANKUPDATE_SPECIALIZE(Scalar) \
-template <typename Index, int LhsStorageOrder, bool ConjugateLhs, \
-                          int RhsStorageOrder, bool ConjugateRhs, int  UpLo> \
-struct general_matrix_matrix_triangular_product<Index,Scalar,LhsStorageOrder,ConjugateLhs, \
-               Scalar,RhsStorageOrder,ConjugateRhs,ColMajor,1,UpLo,Specialized> { \
-  static EIGEN_STRONG_INLINE void run(Index size, Index depth,const Scalar* lhs, Index lhsStride, \
-                          const Scalar* rhs, Index rhsStride, Scalar* res, Index resIncr, Index resStride, Scalar alpha, level3_blocking<Scalar, Scalar>& blocking) \
-  { \
-    if ( lhs==rhs && ((UpLo&(Lower|Upper))==UpLo) ) { \
-      general_matrix_matrix_rankupdate<Index,Scalar,LhsStorageOrder,ConjugateLhs,ColMajor,UpLo> \
-      ::run(size,depth,lhs,lhsStride,rhs,rhsStride,res,resStride,alpha,blocking); \
-    } else { \
-      general_matrix_matrix_triangular_product<Index, \
-        Scalar, LhsStorageOrder, ConjugateLhs, \
-        Scalar, RhsStorageOrder, ConjugateRhs, \
-        ColMajor, 1, UpLo, BuiltIn> \
-      ::run(size,depth,lhs,lhsStride,rhs,rhsStride,res,resIncr,resStride,alpha,blocking); \
-    } \
-  } \
-};
+#define EIGEN_BLAS_RANKUPDATE_SPECIALIZE(Scalar)                                                                      \
+  template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs, int UpLo> \
+  struct general_matrix_matrix_triangular_product<Index, Scalar, LhsStorageOrder, ConjugateLhs, Scalar,               \
+                                                  RhsStorageOrder, ConjugateRhs, ColMajor, 1, UpLo, Specialized> {    \
+    static EIGEN_STRONG_INLINE void run(Index size, Index depth, const Scalar* lhs, Index lhsStride,                  \
+                                        const Scalar* rhs, Index rhsStride, Scalar* res, Index resIncr,               \
+                                        Index resStride, Scalar alpha, level3_blocking<Scalar, Scalar>& blocking) {   \
+      if (lhs == rhs && ((UpLo & (Lower | Upper)) == UpLo)) {                                                         \
+        general_matrix_matrix_rankupdate<Index, Scalar, LhsStorageOrder, ConjugateLhs, ColMajor, UpLo>::run(          \
+            size, depth, lhs, lhsStride, rhs, rhsStride, res, resStride, alpha, blocking);                            \
+      } else {                                                                                                        \
+        general_matrix_matrix_triangular_product<Index, Scalar, LhsStorageOrder, ConjugateLhs, Scalar,                \
+                                                 RhsStorageOrder, ConjugateRhs, ColMajor, 1, UpLo,                    \
+                                                 BuiltIn>::run(size, depth, lhs, lhsStride, rhs, rhsStride, res,      \
+                                                               resIncr, resStride, alpha, blocking);                  \
+      }                                                                                                               \
+    }                                                                                                                 \
+  };
 
 EIGEN_BLAS_RANKUPDATE_SPECIALIZE(double)
 EIGEN_BLAS_RANKUPDATE_SPECIALIZE(float)
@@ -72,74 +72,77 @@ EIGEN_BLAS_RANKUPDATE_SPECIALIZE(float)
 // EIGEN_BLAS_RANKUPDATE_SPECIALIZE(scomplex)
 
 // SYRK for float/double
-#define EIGEN_BLAS_RANKUPDATE_R(EIGTYPE, BLASTYPE, BLASFUNC) \
-template <typename Index, int AStorageOrder, bool ConjugateA, int  UpLo> \
-struct general_matrix_matrix_rankupdate<Index,EIGTYPE,AStorageOrder,ConjugateA,ColMajor,UpLo> { \
-  enum { \
-    IsLower = (UpLo&Lower) == Lower, \
-    LowUp = IsLower ? Lower : Upper, \
-    conjA = ((AStorageOrder==ColMajor) && ConjugateA) ? 1 : 0 \
-  }; \
-  static EIGEN_STRONG_INLINE void run(Index size, Index depth,const EIGTYPE* lhs, Index lhsStride, \
-                          const EIGTYPE* /*rhs*/, Index /*rhsStride*/, EIGTYPE* res, Index resStride, EIGTYPE alpha, level3_blocking<EIGTYPE, EIGTYPE>& /*blocking*/) \
-  { \
-  /* typedef Matrix<EIGTYPE, Dynamic, Dynamic, RhsStorageOrder> MatrixRhs;*/ \
-\
-   BlasIndex lda=convert_index<BlasIndex>(lhsStride), ldc=convert_index<BlasIndex>(resStride), n=convert_index<BlasIndex>(size), k=convert_index<BlasIndex>(depth); \
-   char uplo=((IsLower) ? 'L' : 'U'), trans=((AStorageOrder==RowMajor) ? 'T':'N'); \
-   EIGTYPE beta(1); \
-   BLASFUNC(&uplo, &trans, &n, &k, (const BLASTYPE*)&numext::real_ref(alpha), lhs, &lda, (const BLASTYPE*)&numext::real_ref(beta), res, &ldc); \
-  } \
-};
+#define EIGEN_BLAS_RANKUPDATE_R(EIGTYPE, BLASTYPE, BLASFUNC)                                                        \
+  template <typename Index, int AStorageOrder, bool ConjugateA, int UpLo>                                           \
+  struct general_matrix_matrix_rankupdate<Index, EIGTYPE, AStorageOrder, ConjugateA, ColMajor, UpLo> {              \
+    enum {                                                                                                          \
+      IsLower = (UpLo & Lower) == Lower,                                                                            \
+      LowUp = IsLower ? Lower : Upper,                                                                              \
+      conjA = ((AStorageOrder == ColMajor) && ConjugateA) ? 1 : 0                                                   \
+    };                                                                                                              \
+    static EIGEN_STRONG_INLINE void run(Index size, Index depth, const EIGTYPE* lhs, Index lhsStride,               \
+                                        const EIGTYPE* /*rhs*/, Index /*rhsStride*/, EIGTYPE* res, Index resStride, \
+                                        EIGTYPE alpha, level3_blocking<EIGTYPE, EIGTYPE>& /*blocking*/) {           \
+      /* typedef Matrix<EIGTYPE, Dynamic, Dynamic, RhsStorageOrder> MatrixRhs;*/                                    \
+      if (size == 0 || depth == 0) return;                                                                          \
+      BlasIndex lda = convert_index<BlasIndex>(lhsStride), ldc = convert_index<BlasIndex>(resStride),               \
+                n = convert_index<BlasIndex>(size), k = convert_index<BlasIndex>(depth);                            \
+      char uplo = ((IsLower) ? 'L' : 'U'), trans = ((AStorageOrder == RowMajor) ? 'T' : 'N');                       \
+      EIGTYPE beta(1);                                                                                              \
+      BLASFUNC(&uplo, &trans, &n, &k, (const BLASTYPE*)&numext::real_ref(alpha), lhs, &lda,                         \
+               (const BLASTYPE*)&numext::real_ref(beta), res, &ldc);                                                \
+    }                                                                                                               \
+  };
 
 // HERK for complex data
-#define EIGEN_BLAS_RANKUPDATE_C(EIGTYPE, BLASTYPE, RTYPE, BLASFUNC) \
-template <typename Index, int AStorageOrder, bool ConjugateA, int  UpLo> \
-struct general_matrix_matrix_rankupdate<Index,EIGTYPE,AStorageOrder,ConjugateA,ColMajor,UpLo> { \
-  enum { \
-    IsLower = (UpLo&Lower) == Lower, \
-    LowUp = IsLower ? Lower : Upper, \
-    conjA = (((AStorageOrder==ColMajor) && ConjugateA) || ((AStorageOrder==RowMajor) && !ConjugateA)) ? 1 : 0 \
-  }; \
-  static EIGEN_STRONG_INLINE void run(Index size, Index depth,const EIGTYPE* lhs, Index lhsStride, \
-                          const EIGTYPE* /*rhs*/, Index /*rhsStride*/, EIGTYPE* res, Index resStride, EIGTYPE alpha, level3_blocking<EIGTYPE, EIGTYPE>& /*blocking*/) \
-  { \
-   typedef Matrix<EIGTYPE, Dynamic, Dynamic, AStorageOrder> MatrixType; \
-\
-   BlasIndex lda=convert_index<BlasIndex>(lhsStride), ldc=convert_index<BlasIndex>(resStride), n=convert_index<BlasIndex>(size), k=convert_index<BlasIndex>(depth); \
-   char uplo=((IsLower) ? 'L' : 'U'), trans=((AStorageOrder==RowMajor) ? 'C':'N'); \
-   RTYPE alpha_, beta_; \
-   const EIGTYPE* a_ptr; \
-\
-   alpha_ = alpha.real(); \
-   beta_ = 1.0; \
-/* Copy with conjugation in some cases*/ \
-   MatrixType a; \
-   if (conjA) { \
-     Map<const MatrixType, 0, OuterStride<> > mapA(lhs,n,k,OuterStride<>(lhsStride)); \
-     a = mapA.conjugate(); \
-     lda = a.outerStride(); \
-     a_ptr = a.data(); \
-   } else a_ptr=lhs; \
-   BLASFUNC(&uplo, &trans, &n, &k, &alpha_, (BLASTYPE*)a_ptr, &lda, &beta_, (BLASTYPE*)res, &ldc); \
-  } \
-};
+#define EIGEN_BLAS_RANKUPDATE_C(EIGTYPE, BLASTYPE, RTYPE, BLASFUNC)                                                 \
+  template <typename Index, int AStorageOrder, bool ConjugateA, int UpLo>                                           \
+  struct general_matrix_matrix_rankupdate<Index, EIGTYPE, AStorageOrder, ConjugateA, ColMajor, UpLo> {              \
+    enum {                                                                                                          \
+      IsLower = (UpLo & Lower) == Lower,                                                                            \
+      LowUp = IsLower ? Lower : Upper,                                                                              \
+      conjA = (((AStorageOrder == ColMajor) && ConjugateA) || ((AStorageOrder == RowMajor) && !ConjugateA)) ? 1 : 0 \
+    };                                                                                                              \
+    static EIGEN_STRONG_INLINE void run(Index size, Index depth, const EIGTYPE* lhs, Index lhsStride,               \
+                                        const EIGTYPE* /*rhs*/, Index /*rhsStride*/, EIGTYPE* res, Index resStride, \
+                                        EIGTYPE alpha, level3_blocking<EIGTYPE, EIGTYPE>& /*blocking*/) {           \
+      typedef Matrix<EIGTYPE, Dynamic, Dynamic, AStorageOrder> MatrixType;                                          \
+      if (size == 0 || depth == 0) return;                                                                          \
+      BlasIndex lda = convert_index<BlasIndex>(lhsStride), ldc = convert_index<BlasIndex>(resStride),               \
+                n = convert_index<BlasIndex>(size), k = convert_index<BlasIndex>(depth);                            \
+      char uplo = ((IsLower) ? 'L' : 'U'), trans = ((AStorageOrder == RowMajor) ? 'C' : 'N');                       \
+      RTYPE alpha_, beta_;                                                                                          \
+      const EIGTYPE* a_ptr;                                                                                         \
+                                                                                                                    \
+      alpha_ = alpha.real();                                                                                        \
+      beta_ = 1.0;                                                                                                  \
+      /* Copy with conjugation in some cases*/                                                                      \
+      MatrixType a;                                                                                                 \
+      if (conjA) {                                                                                                  \
+        Map<const MatrixType, 0, OuterStride<> > mapA(lhs, n, k, OuterStride<>(lhsStride));                         \
+        a = mapA.conjugate();                                                                                       \
+        lda = a.outerStride();                                                                                      \
+        a_ptr = a.data();                                                                                           \
+      } else                                                                                                        \
+        a_ptr = lhs;                                                                                                \
+      BLASFUNC(&uplo, &trans, &n, &k, &alpha_, (BLASTYPE*)a_ptr, &lda, &beta_, (BLASTYPE*)res, &ldc);               \
+    }                                                                                                               \
+  };
 
 #ifdef EIGEN_USE_MKL
 EIGEN_BLAS_RANKUPDATE_R(double, double, dsyrk)
-EIGEN_BLAS_RANKUPDATE_R(float,  float,  ssyrk)
+EIGEN_BLAS_RANKUPDATE_R(float, float, ssyrk)
 #else
 EIGEN_BLAS_RANKUPDATE_R(double, double, dsyrk_)
-EIGEN_BLAS_RANKUPDATE_R(float,  float,  ssyrk_)
+EIGEN_BLAS_RANKUPDATE_R(float, float, ssyrk_)
 #endif
 
-// TODO hanlde complex cases
+// TODO handle complex cases
 // EIGEN_BLAS_RANKUPDATE_C(dcomplex, double, double, zherk_)
 // EIGEN_BLAS_RANKUPDATE_C(scomplex, float,  float, cherk_)
 
+}  // end namespace internal
 
-} // end namespace internal
-
-} // end namespace Eigen
+}  // end namespace Eigen
 
-#endif // EIGEN_GENERAL_MATRIX_MATRIX_TRIANGULAR_BLAS_H
+#endif  // EIGEN_GENERAL_MATRIX_MATRIX_TRIANGULAR_BLAS_H

package/eigen/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h

@@ -33,92 +33,173 @@
 #ifndef EIGEN_GENERAL_MATRIX_MATRIX_BLAS_H
 #define EIGEN_GENERAL_MATRIX_MATRIX_BLAS_H
 
-namespace Eigen { 
+// IWYU pragma: private
+#include "../InternalHeaderCheck.h"
+
+namespace Eigen {
 
 namespace internal {
 
 /**********************************************************************
-* This file implements general matrix-matrix multiplication using BLAS
-* gemm function via partial specialization of
-* general_matrix_matrix_product::run(..) method for float, double,
-* std::complex<float> and std::complex<double> types
-**********************************************************************/
+ * This file implements general matrix-matrix multiplication using BLAS
+ * gemm function via partial specialization of
+ * general_matrix_matrix_product::run(..) method for float, double,
+ * std::complex<float> and std::complex<double> types
+ **********************************************************************/
 
 // gemm specialization
 
-#define GEMM_SPECIALIZATION(EIGTYPE, EIGPREFIX, BLASTYPE, BLASFUNC) \
-template< \
-  typename Index, \
-  int LhsStorageOrder, bool ConjugateLhs, \
-  int RhsStorageOrder, bool ConjugateRhs> \
-struct general_matrix_matrix_product<Index,EIGTYPE,LhsStorageOrder,ConjugateLhs,EIGTYPE,RhsStorageOrder,ConjugateRhs,ColMajor,1> \
-{ \
-typedef gebp_traits<EIGTYPE,EIGTYPE> Traits; \
-\
-static void run(Index rows, Index cols, Index depth, \
-  const EIGTYPE* _lhs, Index lhsStride, \
-  const EIGTYPE* _rhs, Index rhsStride, \
-  EIGTYPE* res, Index resIncr, Index resStride, \
-  EIGTYPE alpha, \
-  level3_blocking<EIGTYPE, EIGTYPE>& /*blocking*/, \
-  GemmParallelInfo<Index>* /*info = 0*/) \
-{ \
-  using std::conj; \
-\
-  EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
-  eigen_assert(resIncr == 1); \
-  char transa, transb; \
-  BlasIndex m, n, k, lda, ldb, ldc; \
-  const EIGTYPE *a, *b; \
-  EIGTYPE beta(1); \
-  MatrixX##EIGPREFIX a_tmp, b_tmp; \
-\
-/* Set transpose options */ \
-  transa = (LhsStorageOrder==RowMajor) ? ((ConjugateLhs) ? 'C' : 'T') : 'N'; \
-  transb = (RhsStorageOrder==RowMajor) ? ((ConjugateRhs) ? 'C' : 'T') : 'N'; \
-\
-/* Set m, n, k */ \
-  m = convert_index<BlasIndex>(rows);  \
-  n = convert_index<BlasIndex>(cols);  \
-  k = convert_index<BlasIndex>(depth); \
-\
-/* Set lda, ldb, ldc */ \
-  lda = convert_index<BlasIndex>(lhsStride); \
-  ldb = convert_index<BlasIndex>(rhsStride); \
-  ldc = convert_index<BlasIndex>(resStride); \
-\
-/* Set a, b, c */ \
-  if ((LhsStorageOrder==ColMajor) && (ConjugateLhs)) { \
-    Map<const MatrixX##EIGPREFIX, 0, OuterStride<> > lhs(_lhs,m,k,OuterStride<>(lhsStride)); \
-    a_tmp = lhs.conjugate(); \
-    a = a_tmp.data(); \
-    lda = convert_index<BlasIndex>(a_tmp.outerStride()); \
-  } else a = _lhs; \
-\
-  if ((RhsStorageOrder==ColMajor) && (ConjugateRhs)) { \
-    Map<const MatrixX##EIGPREFIX, 0, OuterStride<> > rhs(_rhs,k,n,OuterStride<>(rhsStride)); \
-    b_tmp = rhs.conjugate(); \
-    b = b_tmp.data(); \
-    ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
-  } else b = _rhs; \
-\
-  BLASFUNC(&transa, &transb, &m, &n, &k, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \
-}};
+#define GEMM_SPECIALIZATION(EIGTYPE, EIGPREFIX, BLASTYPE, BLASFUNC)                                                 \
+  template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs>         \
+  struct general_matrix_matrix_product<Index, EIGTYPE, LhsStorageOrder, ConjugateLhs, EIGTYPE, RhsStorageOrder,     \
+                                       ConjugateRhs, ColMajor, 1> {                                                 \
+    typedef gebp_traits<EIGTYPE, EIGTYPE> Traits;                                                                   \
+                                                                                                                    \
+    static void run(Index rows, Index cols, Index depth, const EIGTYPE* lhs_, Index lhsStride, const EIGTYPE* rhs_, \
+                    Index rhsStride, EIGTYPE* res, Index resIncr, Index resStride, EIGTYPE alpha,                   \
+                    level3_blocking<EIGTYPE, EIGTYPE>& /*blocking*/, GemmParallelInfo<Index>* /*info = 0*/) {       \
+      using std::conj;                                                                                              \
+      if (rows == 0 || cols == 0 || depth == 0) return;                                                             \
+      EIGEN_ONLY_USED_FOR_DEBUG(resIncr);                                                                           \
+      eigen_assert(resIncr == 1);                                                                                   \
+      char transa, transb;                                                                                          \
+      BlasIndex m, n, k, lda, ldb, ldc;                                                                             \
+      const EIGTYPE *a, *b;                                                                                         \
+      EIGTYPE beta(1);                                                                                              \
+      MatrixX##EIGPREFIX a_tmp, b_tmp;                                                                              \
+                                                                                                                    \
+      /* Set transpose options */                                                                                   \
+      transa = (LhsStorageOrder == RowMajor) ? ((ConjugateLhs) ? 'C' : 'T') : 'N';                                  \
+      transb = (RhsStorageOrder == RowMajor) ? ((ConjugateRhs) ? 'C' : 'T') : 'N';                                  \
+                                                                                                                    \
+      /* Set m, n, k */                                                                                             \
+      m = convert_index<BlasIndex>(rows);                                                                           \
+      n = convert_index<BlasIndex>(cols);                                                                           \
+      k = convert_index<BlasIndex>(depth);                                                                          \
+                                                                                                                    \
+      /* Set lda, ldb, ldc */                                                                                       \
+      lda = convert_index<BlasIndex>(lhsStride);                                                                    \
+      ldb = convert_index<BlasIndex>(rhsStride);                                                                    \
+      ldc = convert_index<BlasIndex>(resStride);                                                                    \
+                                                                                                                    \
+      /* Set a, b, c */                                                                                             \
+      if ((LhsStorageOrder == ColMajor) && (ConjugateLhs)) {                                                        \
+        Map<const MatrixX##EIGPREFIX, 0, OuterStride<> > lhs(lhs_, m, k, OuterStride<>(lhsStride));                 \
+        a_tmp = lhs.conjugate();                                                                                    \
+        a = a_tmp.data();                                                                                           \
+        lda = convert_index<BlasIndex>(a_tmp.outerStride());                                                        \
+      } else                                                                                                        \
+        a = lhs_;                                                                                                   \
+                                                                                                                    \
+      if ((RhsStorageOrder == ColMajor) && (ConjugateRhs)) {                                                        \
+        Map<const MatrixX##EIGPREFIX, 0, OuterStride<> > rhs(rhs_, k, n, OuterStride<>(rhsStride));                 \
+        b_tmp = rhs.conjugate();                                                                                    \
+        b = b_tmp.data();                                                                                           \
+        ldb = convert_index<BlasIndex>(b_tmp.outerStride());                                                        \
+      } else                                                                                                        \
+        b = rhs_;                                                                                                   \
+                                                                                                                    \
+      BLASFUNC(&transa, &transb, &m, &n, &k, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda,   \
+               (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc);           \
+    }                                                                                                               \
+  };
 
 #ifdef EIGEN_USE_MKL
-GEMM_SPECIALIZATION(double,   d,  double, dgemm)
-GEMM_SPECIALIZATION(float,    f,  float,  sgemm)
+GEMM_SPECIALIZATION(double, d, double, dgemm)
+GEMM_SPECIALIZATION(float, f, float, sgemm)
 GEMM_SPECIALIZATION(dcomplex, cd, MKL_Complex16, zgemm)
-GEMM_SPECIALIZATION(scomplex, cf, MKL_Complex8,  cgemm)
+GEMM_SPECIALIZATION(scomplex, cf, MKL_Complex8, cgemm)
 #else
-GEMM_SPECIALIZATION(double,   d,  double, dgemm_)
-GEMM_SPECIALIZATION(float,    f,  float,  sgemm_)
+GEMM_SPECIALIZATION(double, d, double, dgemm_)
+GEMM_SPECIALIZATION(float, f, float, sgemm_)
 GEMM_SPECIALIZATION(dcomplex, cd, double, zgemm_)
-GEMM_SPECIALIZATION(scomplex, cf, float,  cgemm_)
+GEMM_SPECIALIZATION(scomplex, cf, float, cgemm_)
 #endif
 
-} // end namespase internal
+// If OpenBLAS with BUILD_BFLOAT16=1 support is available,
+// use sbgemm for bfloat16.
+#if EIGEN_USE_OPENBLAS_BFLOAT16
+
+extern "C" {
+// OpenBLAS prototype.
+void sbgemm_(const char* trans_a, const char* trans_b, const int* M, const int* N, const int* K, const float* alpha,
+             const Eigen::bfloat16* A, const int* lda, const Eigen::bfloat16* B, const int* ldb, const float* beta,
+             float* C, const int* ldc);
+}  // extern "C"
+
+template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs>
+struct general_matrix_matrix_product<Index, Eigen::bfloat16, LhsStorageOrder, ConjugateLhs, Eigen::bfloat16,
+                                     RhsStorageOrder, ConjugateRhs, ColMajor, 1> {
+  typedef gebp_traits<Eigen::bfloat16, Eigen::bfloat16> Traits;
+
+  static void run(Index rows, Index cols, Index depth, const Eigen::bfloat16* lhs_, Index lhsStride,
+                  const Eigen::bfloat16* rhs_, Index rhsStride, Eigen::bfloat16* res, Index resIncr, Index resStride,
+                  Eigen::bfloat16 alpha, level3_blocking<Eigen::bfloat16, Eigen::bfloat16>& /*blocking*/,
+                  GemmParallelInfo<Index>* /*info = 0*/) {
+    using std::conj;
+    if (rows == 0 || cols == 0 || depth == 0) return;
+    EIGEN_ONLY_USED_FOR_DEBUG(resIncr);
+    eigen_assert(resIncr == 1);
+    char transa, transb;
+    BlasIndex m, n, k, lda, ldb, ldc;
+    const Eigen::bfloat16 *a, *b;
+
+    float falpha = static_cast<float>(alpha);
+    float fbeta = float(1.0);
+
+    using MatrixXbf16 = Matrix<Eigen::bfloat16, Dynamic, Dynamic>;
+    MatrixXbf16 a_tmp, b_tmp;
+    MatrixXf r_tmp;
+
+    /* Set transpose options */
+    transa = (LhsStorageOrder == RowMajor) ? ((ConjugateLhs) ? 'C' : 'T') : 'N';
+    transb = (RhsStorageOrder == RowMajor) ? ((ConjugateRhs) ? 'C' : 'T') : 'N';
+
+    /* Set m, n, k */
+    m = convert_index<BlasIndex>(rows);
+    n = convert_index<BlasIndex>(cols);
+    k = convert_index<BlasIndex>(depth);
+
+    /* Set lda, ldb, ldc */
+    lda = convert_index<BlasIndex>(lhsStride);
+    ldb = convert_index<BlasIndex>(rhsStride);
+    ldc = convert_index<BlasIndex>(m);
+
+    /* Set a, b, c */
+    if ((LhsStorageOrder == ColMajor) && (ConjugateLhs)) {
+      Map<const MatrixXbf16, 0, OuterStride<> > lhs(lhs_, m, k, OuterStride<>(lhsStride));
+      a_tmp = lhs.conjugate();
+      a = a_tmp.data();
+      lda = convert_index<BlasIndex>(a_tmp.outerStride());
+    } else {
+      a = lhs_;
+    }
+
+    if ((RhsStorageOrder == ColMajor) && (ConjugateRhs)) {
+      Map<const MatrixXbf16, 0, OuterStride<> > rhs(rhs_, k, n, OuterStride<>(rhsStride));
+      b_tmp = rhs.conjugate();
+      b = b_tmp.data();
+      ldb = convert_index<BlasIndex>(b_tmp.outerStride());
+    } else {
+      b = rhs_;
+    }
+
+    // Evaluate to a temporary intermediate array.
+    r_tmp.resize(m, n);
+
+    sbgemm_(&transa, &transb, &m, &n, &k, (const float*)&numext::real_ref(falpha), a, &lda, b, &ldb,
+            (const float*)&numext::real_ref(fbeta), r_tmp.data(), &ldc);
+
+    // Cast to the output.
+    Map<MatrixXbf16, 0, OuterStride<> > result(res, m, n, OuterStride<>(resStride));
+    result = r_tmp.cast<Eigen::bfloat16>();
+  }
+};
+
+#endif  // EIGEN_USE_OPENBLAS_SBGEMM
+
+}  // namespace internal
 
-} // end namespace Eigen
+}  // end namespace Eigen
 
-#endif // EIGEN_GENERAL_MATRIX_MATRIX_BLAS_H
+#endif  // EIGEN_GENERAL_MATRIX_MATRIX_BLAS_H