alglib4 0.0.0

data/ext/alglib/kernels_avx2.cpp

@@ -0,0 +1,2171 @@
+/*************************************************************************
+ALGLIB 4.04.0 (source code generated 2024-12-21)
+Copyright (c) Sergey Bochkanov (ALGLIB project).
+
+>>> SOURCE LICENSE >>>
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation (www.fsf.org); either version 2 of the 
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+A copy of the GNU General Public License is available at
+http://www.fsf.org/licensing/licenses
+>>> END OF LICENSE >>>
+*************************************************************************/
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+#include "stdafx.h"
+
+//
+// Must be defined before we include kernel header
+//
+#define _ALGLIB_IMPL_DEFINES
+#define _ALGLIB_INTEGRITY_CHECKS_ONCE
+
+#include "kernels_avx2.h"
+
+// disable some irrelevant warnings
+#if (AE_COMPILER==AE_MSVC) && !defined(AE_ALL_WARNINGS)
+#pragma warning(disable:4100)
+#pragma warning(disable:4127)
+#pragma warning(disable:4611)
+#pragma warning(disable:4702)
+#pragma warning(disable:4996)
+#endif
+
+namespace alglib_impl
+{
+
+
+
+#if !defined(ALGLIB_NO_FAST_KERNELS) && defined(_ALGLIB_HAS_AVX2_INTRINSICS)
+
+double rdotv_avx2(const ae_int_t n,
+    /* Real    */ const double* __restrict x,
+    /* Real    */ const double* __restrict y,
+    const ae_state* __restrict _state)
+{
+    ae_int_t i;
+
+    const ae_int_t avx2len = n>>2;
+    const ae_int_t fmaLen = (avx2len >> 2) << 2;
+    const __m256d* __restrict pX = (const __m256d*)(x);
+    const __m256d* __restrict pY = (const __m256d*)(y);
+    __m256d ans;
+    if(fmaLen >= 4) {
+        __m256d fmaUnroll[4];
+        fmaUnroll[0] = _mm256_mul_pd(pX[0], pY[0]);
+        fmaUnroll[1] = _mm256_mul_pd(pX[1], pY[1]);
+        fmaUnroll[2] = _mm256_mul_pd(pX[2], pY[2]);
+        fmaUnroll[3] = _mm256_mul_pd(pX[3], pY[3]);
+        for(i=4; i<fmaLen; i+=4) {
+            fmaUnroll[0] = _mm256_add_pd(_mm256_mul_pd(pX[i], pY[i]), fmaUnroll[0]);
+            fmaUnroll[1] = _mm256_add_pd(_mm256_mul_pd(pX[i+1], pY[i+1]), fmaUnroll[1]);
+            fmaUnroll[2] = _mm256_add_pd(_mm256_mul_pd(pX[i+2], pY[i+2]), fmaUnroll[2]);
+            fmaUnroll[3] = _mm256_add_pd(_mm256_mul_pd(pX[i+3], pY[i+3]), fmaUnroll[3]);
+        }
+        switch(avx2len-fmaLen) {
+        case 3:
+            fmaUnroll[2] = _mm256_add_pd(_mm256_mul_pd(pX[i+2], pY[i+2]), fmaUnroll[2]);
+        case 2:
+            fmaUnroll[1] = _mm256_add_pd(_mm256_mul_pd(pX[i+1], pY[i+1]), fmaUnroll[1]);
+        case 1:
+            fmaUnroll[0] = _mm256_add_pd(_mm256_mul_pd(pX[i], pY[i]), fmaUnroll[0]);
+        }
+        ans = _mm256_add_pd(
+            _mm256_add_pd(fmaUnroll[0], fmaUnroll[1]),
+            _mm256_add_pd(fmaUnroll[2], fmaUnroll[3]));
+    }
+    else {
+        ans = _mm256_setzero_pd();
+        switch(avx2len) {
+        case 3:
+            ans = _mm256_mul_pd(pX[2], pY[2]);
+        case 2:
+            ans = _mm256_add_pd(_mm256_mul_pd(pX[1], pY[1]), ans);
+        case 1:
+            ans = _mm256_add_pd(_mm256_mul_pd(pX[0], pY[0]), ans);
+        }
+    }
+    const __m128d s = _mm_add_pd(_mm256_extractf128_pd(ans, 0), _mm256_extractf128_pd(ans, 1));
+    const double *pComps = (const double*)&s;
+    double dot =pComps[0] + pComps[1];
+    const ae_int_t tail = avx2len<<2;
+    switch(n-tail) {
+    case 1: {
+        dot += x[tail]*y[tail];
+        break;
+    }
+    case 2: {
+        dot += x[tail+0]*y[tail+0];
+        dot += x[tail+1]*y[tail+1];
+        break;
+    }
+    case 3: {
+        dot += x[tail+0]*y[tail+0];
+        dot += x[tail+1]*y[tail+1];
+        dot += x[tail+2]*y[tail+2];
+        break;
+    }
+    }
+    return dot;
+}
+
+double rdotv2_avx2(const ae_int_t n,
+    /* Real    */ const double* __restrict x,
+    const ae_state* __restrict _state)
+{
+    ae_int_t i;
+
+    const ae_int_t avx2len = n>>2;
+    const ae_int_t fmaLen = (avx2len >> 2) << 2;
+    const __m256d* __restrict pX = (const __m256d*)(x);
+    __m256d ans;
+    if(fmaLen >= 4) {
+        //TODO: this can be further unrolled to 8 because AVX(2) provides 16 registers
+        __m256d fmaUnroll[4];
+        fmaUnroll[0] = _mm256_mul_pd(pX[0], pX[0]);
+        fmaUnroll[1] = _mm256_mul_pd(pX[1], pX[1]);
+        fmaUnroll[2] = _mm256_mul_pd(pX[2], pX[2]);
+        fmaUnroll[3] = _mm256_mul_pd(pX[3], pX[3]);
+        for(i=4; i<fmaLen; i+=4) {
+            fmaUnroll[0] = _mm256_add_pd(_mm256_mul_pd(pX[i], pX[i]), fmaUnroll[0]);
+            fmaUnroll[1] = _mm256_add_pd(_mm256_mul_pd(pX[i+1], pX[i+1]), fmaUnroll[1]);
+            fmaUnroll[2] = _mm256_add_pd(_mm256_mul_pd(pX[i+2], pX[i+2]), fmaUnroll[2]);
+            fmaUnroll[3] = _mm256_add_pd(_mm256_mul_pd(pX[i+3], pX[i+3]), fmaUnroll[3]);
+        }
+        switch(avx2len-fmaLen) {
+        case 3:
+            fmaUnroll[2] = _mm256_add_pd(_mm256_mul_pd(pX[i+2], pX[i+2]), fmaUnroll[2]);
+        case 2:
+            fmaUnroll[1] = _mm256_add_pd(_mm256_mul_pd(pX[i+1], pX[i+1]), fmaUnroll[1]);
+        case 1:
+            fmaUnroll[0] = _mm256_add_pd(_mm256_mul_pd(pX[i], pX[i]), fmaUnroll[0]);
+        }
+        ans = _mm256_add_pd(
+            _mm256_add_pd(fmaUnroll[0], fmaUnroll[1]),
+            _mm256_add_pd(fmaUnroll[2], fmaUnroll[3]));
+    }
+    else {
+        ans = _mm256_setzero_pd();
+        switch(avx2len) {
+        case 3:
+            ans = _mm256_mul_pd(pX[2], pX[2]);
+        case 2:
+            ans = _mm256_add_pd(_mm256_mul_pd(pX[1], pX[1]), ans);
+        case 1:
+            ans = _mm256_add_pd(_mm256_mul_pd(pX[0], pX[0]), ans);
+        }
+    }
+    const __m128d s = _mm_add_pd(_mm256_extractf128_pd(ans, 0), _mm256_extractf128_pd(ans, 1));
+    const double *pComps = (const double*)&s;
+    double dot =pComps[0] + pComps[1];
+    const ae_int_t tail = avx2len<<2;
+    switch(n-tail) {
+    case 1: {
+        dot += x[tail]*x[tail];
+        break;
+    }
+    case 2: {
+        dot += x[tail+0]*x[tail+0];
+        dot += x[tail+1]*x[tail+1];
+        break;
+    }
+    case 3: {
+        dot += x[tail+0]*x[tail+0];
+        dot += x[tail+1]*x[tail+1];
+        dot += x[tail+2]*x[tail+2];
+        break;
+    }
+    }
+    return dot;
+}
+
+void rcopyv_avx2(ae_int_t n,
+     /* Real    */ const double* __restrict x,
+     /* Real    */ double* __restrict y,
+     ae_state* __restrict _state)
+{
+    ae_int_t i;
+
+    const ae_int_t avx2len = n>>2;
+    const ae_int_t tail = avx2len<<2;
+    const __m256d* __restrict pSrc = (const __m256d*)(x);
+    __m256d* __restrict pDest = (__m256d*)(y);
+    for(i=0; i<avx2len; i++) {
+        pDest[i] = pSrc[i];
+    }
+    switch(n-tail) {
+    case 1:
+        *(double*)(pDest+i) = *(const double*)(pSrc+i);
+        break;
+    case 2:
+        *(__m128d*)(pDest+i) = *(const __m128d*)(pSrc+i);
+        break;
+    case 3:
+        *(__m128d*)(pDest+i) = *(const __m128d*)(pSrc+i);
+        y[tail+2] = x[tail+2];
+        break;
+    }
+}
+
+void rcopymulv_avx2(const ae_int_t n,
+     const double v,
+     /* Real    */ const double* __restrict x,
+     /* Real    */ double* __restrict y,
+     const ae_state* __restrict _state)
+{
+    ae_int_t i;
+    
+    const ae_int_t avx2len = n>>2;
+    const ae_int_t tail = avx2len<<2;
+    const __m256d* __restrict pSrc = (const __m256d*)(x);
+    __m256d* __restrict pDest = (__m256d*)(y);
+    const __m256d avx2v = _mm256_set1_pd(v);
+    for(i=0; i<avx2len; i++) {
+        pDest[i] = _mm256_mul_pd(avx2v, pSrc[i]);
+    }
+    switch(n-tail) {
+    case 1:
+        *(double*)(pDest+i) = v * (*(const double*)(pSrc+i));
+        break;
+    case 2:
+        *(__m128d*)(pDest+i) = _mm_mul_pd(_mm256_extractf128_pd(avx2v, 0), *(const __m128d*)(pSrc+i));
+        break;
+    case 3:
+        *(__m128d*)(pDest+i) = _mm_mul_pd(_mm256_extractf128_pd(avx2v, 0), *(const __m128d*)(pSrc+i));
+        y[tail+2] = v*x[tail+2];
+        break;
+    }
+}
+
+void icopyv_avx2(const ae_int_t n, const ae_int_t* __restrict x,
+                ae_int_t* __restrict y, ae_state* __restrict _state)
+{
+    const ae_int_t tail = (n*sizeof(ae_int_t)) & 31;
+    const ae_int_t even = (n*sizeof(ae_int_t)) - tail;
+    __m256i *__restrict pDest = (__m256i*)y;
+    const __m256i* __restrict pSrc = (const __m256i*)x;
+    const ae_int_t nVec = even>>5;
+    ae_int_t i;
+    for(i=0; i<nVec; i++) {
+        pDest[i] = pSrc[i];
+    }
+    i = even/sizeof(ae_int_t);
+    if(tail & 16) {
+        *(__m128i*)(y+i) = *(const __m128i*)(x+i);
+        i += 16/sizeof(ae_int_t);
+    }
+    if(tail & 8) {
+        *(ae_int64_t*)(y+i) = *(const ae_int64_t*)(x+i);
+        i += 8/sizeof(ae_int_t);
+    }
+    if(tail & 4) {
+        *(ae_int32_t*)(y+i) = *(const ae_int32_t*)(x+i);
+    }
+}
+
+void bcopyv_avx2(const ae_int_t n, const ae_bool* __restrict x,
+                ae_bool* __restrict y, ae_state* __restrict _state)
+{
+    const ae_int_t tail = n & 31;
+    const ae_int_t even = n-tail;
+    __m256i *__restrict pDest = (__m256i*)y;
+    const __m256i* __restrict pSrc = (const __m256i*)x;
+    const ae_int_t nVec = even>>5;
+    ae_int_t i;
+    for(i=0; i<nVec; i++) {
+        pDest[i] = pSrc[i];
+    }
+    i = even;
+    if(tail & 16) {
+        *(__m128i*)(y+i) = *(const __m128i*)(x+i);
+        i += 16;
+    }
+    if(tail & 8) {
+        *(ae_int64_t*)(y+i) = *(const ae_int64_t*)(x+i);
+        i += 8;
+    }
+    if(tail & 4) {
+        *(ae_int32_t*)(y+i) = *(const ae_int32_t*)(x+i);
+        i += 4;
+    }
+    if(tail & 2) {
+        *(y+i+0) = *(x+i+0);
+        *(y+i+1) = *(x+i+1);
+        i += 2;
+    }
+    if(tail & 1) {
+        *(y+i) = *(x+i);
+    }
+}
+
+void rsetv_avx2(const ae_int_t n,
+     const double v,
+     /* Real    */ double* __restrict x,
+     const ae_state* __restrict _state)
+{
+    ae_int_t i;
+    
+    const ae_int_t avx2len = n>>2;
+    __m256d* __restrict pDest = (__m256d*)(x);
+    const __m256d avx2v = _mm256_set1_pd(v);
+    for(i=0; i<avx2len; i++) {
+        pDest[i] = avx2v;
+    }
+    const ae_int_t tail = avx2len<<2;
+    switch(n-tail) {
+    case 1:
+        *(double*)(pDest+i) = v;
+        break;
+    case 2:
+        *(__m128d*)(pDest+i) = _mm256_extractf128_pd(avx2v, 0);
+        break;
+    case 3: 
+        *(__m128d*)(pDest+i) = _mm256_extractf128_pd(avx2v, 0);
+        x[tail+2] = v;
+        break;
+    }
+}
+
+void rsetvx_avx2(const ae_int_t n, const double v, double* __restrict x,
+    const ae_state* __restrict _state)
+{
+    const ptrdiff_t unal = ((ptrdiff_t)x) & 31;
+    if( n<=4 )
+    {
+        ae_int_t j;
+        for(j=0; j<=n-1; j++)
+            x[j] = v;
+        return;
+    }
+    switch(unal)
+    {
+        case 0:
+            rsetv_avx2(n, v, x, _state);
+            return;
+        case 8:
+            x[2] = v;
+        case 16:
+            x[1] = v;
+        case 24:
+        {
+            x[0] = v;
+            const ptrdiff_t nDone = 4-(unal>>3);
+            rsetv_avx2(n-nDone, v, x+nDone, _state);
+            return;
+        }
+    }
+}
+
+void isetv_avx2(const ae_int_t n, const ae_int_t v,
+    ae_int_t* __restrict x, ae_state* __restrict _state)
+{
+    const ae_int_t tail = (n*sizeof(ae_int_t)) & 31;
+    const ae_int_t even = (n*sizeof(ae_int_t)) - tail;
+    __m256i *__restrict pDest = (__m256i*)x;
+    const __m256i avx2v = ((sizeof(v) == 4) ? _mm256_set1_epi32((ae_int32_t)v) : _mm256_set1_epi64x(v));
+    const ae_int_t nVec = even>>5;
+    ae_int_t i;
+    for(i=0; i<nVec; i++) {
+        pDest[i] = avx2v;
+    }
+    memmove(pDest+i, &avx2v, tail);
+}
+
+void bsetv_avx2(const ae_int_t n, const ae_bool v, ae_bool* __restrict x,
+    ae_state* __restrict _state)
+{
+    const ae_int_t tail = n & 31;
+    const ae_int_t even = n-tail;
+    __m256i *__restrict pDest = (__m256i*)x;
+    const __m256i avx2v = _mm256_set1_epi8(v);
+    const ae_int_t nVec = even>>5;
+    ae_int_t i;
+    for(i=0; i<nVec; i++) {
+        pDest[i] = avx2v;
+    }
+    /* _mm256_extracti128_si256() has a too high latency on latest processors (Skylake+) */
+    memset(x+even, v, tail);
+}
+
+void rmulv_avx2(const ae_int_t n, const double v, double* __restrict x,
+     const ae_state* __restrict _state)
+{
+    ae_int_t i;
+    
+    const ae_int_t avx2len = n>>2;
+    __m256d* __restrict pDest = (__m256d*)(x);
+    const __m256d avx2v = _mm256_set1_pd(v);
+    for(i=0; i<avx2len; i++) {
+        pDest[i] = _mm256_mul_pd(avx2v, pDest[i]);
+    }
+    const ae_int_t tail = avx2len<<2;
+    switch(n-tail) {
+    case 1:
+        *(double*)(pDest+i) = v * (*(const double*)(pDest+i));
+        break;
+    case 2:
+        *(__m128d*)(pDest+i) = _mm_mul_pd(_mm256_extractf128_pd(avx2v, 0), *(const __m128d*)(pDest+i));
+        break;
+    case 3:
+        *(__m128d*)(pDest+i) = _mm_mul_pd(_mm256_extractf128_pd(avx2v, 0), *(const __m128d*)(pDest+i));
+        x[tail+2] *= v;
+        break;
+    }
+}
+
+void rsqrtv_avx2(const ae_int_t n, double* __restrict x, const ae_state* __restrict _state)
+{
+    ae_int_t i;
+    
+    const ae_int_t avx2len = n>>2;
+    const ae_int_t tail = avx2len<<2;
+    __m256d* __restrict pDest = (__m256d*)(x);
+    for(i=0; i<avx2len; i++)
+        pDest[i] = _mm256_sqrt_pd(pDest[i]);
+    for(i=tail; i<n; i++)
+        x[i] = sqrt(x[i]);
+}
+
+void rmulvx_avx2(const ae_int_t n, const double v, double* __restrict x,
+    const ae_state* __restrict _state)
+{
+    const ptrdiff_t unal = ((ptrdiff_t)x) & 31;
+    if( n<=4 )
+    {
+        ae_int_t i;
+        for(i=0; i<=n-1; i++)
+            x[i] *= v;
+        return;
+    }
+    switch(unal) {
+    case 0:
+        rmulv_avx2(n, v, x, _state);
+        return;
+    case 8:
+        x[2] = v*x[2];
+    case 16:
+        x[1] = v*x[1];
+    case 24: {
+        x[0] = v*x[0];
+        const ptrdiff_t nDone = 4-(unal>>3);
+        rmulv_avx2(n-nDone, v, x+nDone, _state);
+        return;
+    }
+    }
+}
+
+void raddv_avx2(const ae_int_t n,
+     const double alpha,
+     /* Real    */ const double* __restrict y,
+     /* Real    */ double* __restrict x,
+     const ae_state* __restrict _state)
+{
+    ae_int_t i;
+    
+    const ae_int_t avx2len = n>>2;
+    const __m256d* __restrict pSrc = (const __m256d*)(y);
+    __m256d* __restrict pDest = (__m256d*)(x);
+    const __m256d avx2alpha = _mm256_set1_pd(alpha);
+    for(i=0; i<avx2len; i++) {
+        pDest[i] = _mm256_add_pd(_mm256_mul_pd(avx2alpha, pSrc[i]), pDest[i]);
+    }
+    const ae_int_t tail = avx2len<<2;
+    switch(n-tail) {
+    case 1:
+        *(double*)(pDest+i) = alpha * (*(const double*)(pSrc+i))
+            + (*(const double*)(pDest+i));
+        break;
+    case 2:
+        *(__m128d*)(pDest+i) = _mm_add_pd(_mm_mul_pd(_mm256_extractf128_pd(avx2alpha, 0), *(const __m128d*)(pSrc+i)),*(const __m128d*)(pDest+i));
+        break;
+    case 3:
+        *(__m128d*)(pDest+i) = _mm_add_pd(_mm_mul_pd(_mm256_extractf128_pd(avx2alpha, 0), *(const __m128d*)(pSrc+i)),*(const __m128d*)(pDest+i));
+        x[tail+2] += alpha*y[tail+2];
+        break;
+    }
+}
+
+void raddvx_avx_xaligned(const ae_int_t n, const double alpha, const double* __restrict y, double* __restrict x, ae_state *_state)
+{
+    ae_int_t i;
+    const ae_int_t vecLen = (n>>2)<<2;
+    const __m256d avx2alpha = _mm256_set1_pd(alpha);
+    __m256d* __restrict pDest = (__m256d*)x;
+    for(i=0; i<vecLen; i+=4)
+    {
+        const ae_int_t iDest = i>>2;
+        pDest[iDest] = _mm256_add_pd(_mm256_mul_pd(avx2alpha, _mm256_loadu_pd(y+i)), pDest[iDest]);
+    }
+    switch(n-vecLen) {
+    case 1:
+        x[i] += alpha*y[i];
+        break;
+    case 2: {
+        const ae_int_t iDest = i>>2;
+        *(__m128d*)(pDest+iDest) = _mm_add_pd(_mm_mul_pd(_mm256_extractf128_pd(avx2alpha, 0),_mm_loadu_pd(y+i)),*(const __m128d*)(pDest+iDest));
+        break;
+    }
+    case 3:
+    {
+        const ae_int_t iDest = i>>2;
+        *(__m128d*)(pDest+iDest) = _mm_add_pd(_mm_mul_pd(_mm256_extractf128_pd(avx2alpha, 0),_mm_loadu_pd(y+i)),*(const __m128d*)(pDest+iDest));
+        x[i+2] += alpha*y[i+2];
+        break;
+    }
+    }
+}
+
+void raddvx_avx2(const ae_int_t n, const double alpha, const double* __restrict y,
+    double* __restrict x, ae_state *_state)
+{
+    const ptrdiff_t unal = ((ptrdiff_t)x) & 31;
+    if( n<=4 )
+    {
+        ae_int_t i;
+        for(i=0; i<=n-1; i++)
+            x[i] += alpha*y[i];
+        return;
+    }
+    switch(unal)
+    {
+    case 0:
+        raddvx_avx_xaligned(n, alpha, y, x, _state);
+        return;
+    case 8:
+        x[2] += alpha*y[2];
+    case 16:
+        x[1] += alpha*y[1];
+    case 24:
+        {
+            x[0] += alpha*y[0];
+            const ptrdiff_t nDone = 4-(unal>>3);
+            raddvx_avx_xaligned(n-nDone, alpha, y+nDone, x+nDone, _state);
+            return;
+        }
+    }
+}
+
+void rmergemulv_avx2(ae_int_t n,
+     /* Real    */ const double* __restrict y,
+     /* Real    */ double* __restrict x,
+     const ae_state* __restrict _state)
+{
+    ae_int_t i;
+
+    const ae_int_t avx2len = n>>2;
+    const __m256d* __restrict pSrc = (const __m256d*)(y);
+    __m256d* __restrict pDest = (__m256d*)(x);
+    for(i=0; i<avx2len; i++) {
+        pDest[i] = _mm256_mul_pd(pSrc[i], pDest[i]);
+    }
+    const ae_int_t tail = avx2len<<2;
+    switch(n-tail) {
+    case 1:
+        *(double*)(pDest+i) = *(const double*)(pSrc+i)
+            * (*(const double*)(pDest+i));
+        break;
+    case 2:
+        *(__m128d*)(pDest+i) = _mm_mul_pd(
+            *(const __m128d*)(pSrc+i), *(const __m128d*)(pDest+i));
+        break;
+    case 3: {
+        *(__m128d*)(pDest+i) = _mm_mul_pd(*(const __m128d*)(pSrc+i), *(const __m128d*)(pDest+i));
+        ((double*)(pDest+i))[2] *= ((const double*)(pSrc+i))[2];
+        break;
+    }
+    }
+}
+
+void rmergedivv_avx2(ae_int_t n,
+     /* Real    */ const double* __restrict y,
+     /* Real    */ double* __restrict x,
+     const ae_state* __restrict _state)
+{
+    ae_int_t i;
+
+    const ae_int_t avx2len = n>>2;
+    const __m256d* __restrict pSrc = (const __m256d*)(y);
+    __m256d* __restrict pDest = (__m256d*)(x);
+    for(i=0; i<avx2len; i++) {
+        pDest[i] = _mm256_div_pd(pDest[i], pSrc[i]);
+    }
+    const ae_int_t tail = avx2len<<2;
+    switch(n-tail) {
+    case 1:
+        *(double*)(pDest+i) = (*(const double*)(pDest+i)) / (*(const double*)(pSrc+i));
+        break;
+    case 2:
+        *(__m128d*)(pDest+i) = _mm_div_pd(*(const __m128d*)(pDest+i),  *(const __m128d*)(pSrc+i));
+        break;
+    case 3: {
+        *(__m128d*)(pDest+i) = _mm_div_pd(*(const __m128d*)(pDest+i), *(const __m128d*)(pSrc+i));
+        ((double*)(pDest+i))[2] /= ((const double*)(pSrc+i))[2];
+        break;
+    }
+    }
+}
+
+void rmergemaxv_avx2(ae_int_t n,
+     /* Real    */ const double* __restrict y,
+     /* Real    */ double* __restrict x,
+     ae_state* __restrict _state)
+{
+    ae_int_t i;
+
+    const ae_int_t avx2len = n>>2;
+    const __m256d* __restrict pSrc = (const __m256d*)(y);
+    __m256d* __restrict pDest = (__m256d*)(x);
+    for(i=0; i<avx2len; i++) {
+        pDest[i] = _mm256_max_pd(pSrc[i], pDest[i]);
+    }
+    const ae_int_t tail = avx2len<<2;
+    switch(n-tail) {
+    case 1:
+        *(double*)(pDest+i) = *(const double*)(pSrc+i)>*(const double*)(pDest+i) ? *(const double*)(pSrc+i) : *(const double*)(pDest+i);
+        break;
+    case 2:
+        *(__m128d*)(pDest+i) = _mm_max_pd(*(const __m128d*)(pSrc+i), *(const __m128d*)(pDest+i));
+        break;
+    case 3:
+    {
+        double  s2 = ((const double*)(pSrc+i))[2];
+        double *d2 = ((double*)(pDest+i))+2;
+        *(__m128d*)(pDest+i) = _mm_max_pd(*(const __m128d*)(pSrc+i), *(const __m128d*)(pDest+i));
+        *d2 = s2>*d2 ? s2 : *d2;
+        break;
+    }
+    }
+}
+
+void rmergeminv_avx2(ae_int_t n,
+     /* Real    */ const double* __restrict y,
+     /* Real    */ double* __restrict x,
+     ae_state* __restrict _state)
+{
+    ae_int_t i;
+
+    const ae_int_t avx2len = n>>2;
+    const __m256d* __restrict pSrc = (const __m256d*)(y);
+    __m256d* __restrict pDest = (__m256d*)(x);
+    for(i=0; i<avx2len; i++) {
+        pDest[i] = _mm256_min_pd(pSrc[i], pDest[i]);
+    }
+    const ae_int_t tail = avx2len<<2;
+    switch(n-tail) {
+    case 1:
+        *(double*)(pDest+i) = ae_minreal(*(const double*)(pSrc+i),
+            *(const double*)(pDest+i), _state);
+        break;
+    case 2:
+        *(__m128d*)(pDest+i) = _mm_min_pd(
+            *(const __m128d*)(pSrc+i), *(const __m128d*)(pDest+i));
+        break;
+    case 3: {
+        double  s2 = ((const double*)(pSrc+i))[2];
+        double *d2 = ((double*)(pDest+i))+2;
+        *(__m128d*)(pDest+i) = _mm_min_pd(*(const __m128d*)(pSrc+i), *(const __m128d*)(pDest+i));
+        *d2 = s2<*d2 ? s2 : *d2;
+        break;
+    }
+    }
+}
+
+double rmaxv_avx2(ae_int_t n, /* Real    */ const double* __restrict x, ae_state* __restrict _state)
+{
+    ae_int_t i;
+    const ae_int_t avx2len = n>>2;
+    const ae_int_t tail = avx2len<<2;
+    const __m256d* __restrict pSrc = (const __m256d*)(x);
+    if( n<=4 )
+    {
+        double result;
+        if(n == 0)
+            return 0.0;
+        result = x[0];
+        for(i=1; i<=n-1; i++)
+        {
+            double v = x[i];
+            if( v>result )
+                result = v;
+        }
+        return result;
+    }
+    __m256d curMax = pSrc[0];
+    for(i=1; i<avx2len; i++) {
+        curMax = _mm256_max_pd(curMax, pSrc[i]);
+    }
+    const __m128d sseMax = _mm_max_pd(_mm256_extractf128_pd(curMax, 0), _mm256_extractf128_pd(curMax, 1));
+    const double *pComps = (const double *)&sseMax;
+    double dMax = (pComps[0] > pComps[1]) ? pComps[0] : pComps[1];
+    const double *p_tail = (const double*)(pSrc+i);
+    switch(n-tail)
+    {
+        case 1:
+        {
+            dMax = p_tail[0]>dMax ? p_tail[0] : dMax;
+            break;
+        }
+        case 2: {
+            dMax = p_tail[0]>dMax ? p_tail[0] : dMax;
+            dMax = p_tail[1]>dMax ? p_tail[1] : dMax;
+            break;
+        }
+        case 3: {
+            dMax = p_tail[0]>dMax ? p_tail[0] : dMax;
+            dMax = p_tail[1]>dMax ? p_tail[1] : dMax;
+            dMax = p_tail[2]>dMax ? p_tail[2] : dMax;
+            break;
+        }
+    }
+    return dMax;
+}
+
+double rmaxabsv_avx2(ae_int_t n, /* Real    */ const double* __restrict x, ae_state* __restrict _state)
+{
+    const __m256d signMask = _mm256_set1_pd(-0.); // -0. = 1 << 63
+    const ae_int_t avx2len = n>>2;
+    const __m256d* __restrict pSrc = (const __m256d*)(x);
+    if( n<=4 )
+    {
+        double result;
+        ae_int_t i;
+        result = 0;
+        for(i=0; i<=n-1; i++)
+        {
+            double v = fabs(x[i]);
+            if( v>result )
+                result = v;
+        }
+        return result;
+    }
+    __m256d curMax = _mm256_andnot_pd(signMask, pSrc[0]); // abs
+    ae_int_t i;
+    for(i=1; i<avx2len; i++) {
+        curMax = _mm256_max_pd(curMax, _mm256_andnot_pd(signMask, pSrc[i])); // abs
+    }
+    const ae_int_t tail = avx2len<<2;
+    const __m128d sseMax = _mm_max_pd(_mm256_extractf128_pd(curMax, 0), _mm256_extractf128_pd(curMax, 1));
+    const double *p_tail = (const double*)(pSrc+i);
+    const double *pComps = (const double *)&sseMax;
+    double dMax = (pComps[0] > pComps[1]) ? pComps[0] : pComps[1];
+    switch(n-tail)
+    {
+        case 1:
+        {
+            double a0 = fabs(p_tail[0]);
+            dMax = a0>dMax ? a0 : dMax;
+            break;
+        }
+        case 2:
+        {
+            double a0 = fabs(p_tail[0]);
+            double a1 = fabs(p_tail[1]);
+            dMax = a0>dMax ? a0 : dMax;
+            dMax = a1>dMax ? a1 : dMax;
+            break;
+        }
+        case 3:
+        {
+            double a0 = fabs(p_tail[0]);
+            double a1 = fabs(p_tail[1]);
+            double a2 = fabs(p_tail[2]);
+            dMax = a0>dMax ? a0 : dMax;
+            dMax = a1>dMax ? a1 : dMax;
+            dMax = a2>dMax ? a2 : dMax;
+            break;
+        }
+    }
+    return dMax;
+}
+
+static void rcopyvx_avx2_xaligned(const ae_int_t n, const double* __restrict x,
+    double* __restrict y, ae_state *_state)
+{
+    ae_int_t i;
+    const ae_int_t vecLen = (n>>2)<<2;
+    const __m256d* __restrict pSrc = (const __m256d*)x;
+    for(i=0; i<vecLen; i+=4) {
+        const ae_int_t iSrc = i>>2;
+        _mm256_storeu_pd(y+i, pSrc[iSrc]);
+    }
+    switch(n-vecLen) {
+    case 1:
+        y[i] = x[i];
+        break;
+    case 2: {
+        const ae_int_t iSrc = i>>2;
+        _mm_storeu_pd(y+i, *(const __m128d*)(pSrc+iSrc));
+        break;
+    }
+    case 3: {
+        const ae_int_t iSrc = i>>2;
+        const __m256d t = pSrc[iSrc];
+        _mm_storeu_pd(y+i, _mm256_extractf128_pd(t, 0));
+        y[i+2] = *(((const double*)&t)+2);
+        break;
+    }
+    }
+}
+
+void rcopyvx_avx2(const ae_int_t n, const double* __restrict x,
+    double* __restrict y, ae_state *_state)
+{
+    const ptrdiff_t unal = ((ptrdiff_t)x) & 31;
+    if( n<=4 )
+    {
+        ae_int_t j;
+        for(j=0; j<n; j++)
+            y[j] = x[j];
+        return;
+    }
+    switch(unal) {
+    case 0:
+        rcopyvx_avx2_xaligned(n, x, y, _state);
+        return;
+    case 8:
+        y[2] = x[2];
+    case 16:
+        y[1] = x[1];
+    case 24: {
+        y[0] = x[0];
+        const ptrdiff_t nDone = 4-(unal>>3);
+        rcopyvx_avx2_xaligned(n-nDone, x+nDone, y+nDone, _state);
+        return;
+    }
+    }
+}
+
+static void icopyvx_avx2_xaligned(const ae_int_t n, const ae_int_t* __restrict x,
+                ae_int_t* __restrict y, ae_state* __restrict _state)
+{
+    const ae_int_t tail = (n*sizeof(ae_int_t)) & 31;
+    const ae_int_t even = (n*sizeof(ae_int_t)) - tail;
+    const __m256i* __restrict pSrc = (const __m256i*)x;
+    const ae_int_t nVec = even>>5;
+    const ae_int_t shift_by = 3-sizeof(ae_int_t)/8;
+    ae_int_t i;
+    for(i=0; i<nVec; i++) {
+        const ae_int_t j = i<<shift_by;
+        _mm256_storeu_si256((__m256i*)(y+j), pSrc[i]);
+    }
+    i = even/sizeof(ae_int_t);
+    if(tail & 16) {
+        _mm_storeu_si128((__m128i*)(y+i), *(const __m128i*)(x+i));
+        i += 16/sizeof(ae_int_t);
+    }
+    if(tail & 8) {
+        *(ae_int64_t*)(y+i) = *(const ae_int64_t*)(x+i);
+        i += 8/sizeof(ae_int_t);
+    }
+    if(tail & 4) {
+        *(ae_int32_t*)(y+i) = *(const ae_int32_t*)(x+i);
+    }
+}
+
+void icopyvx_avx2(const ae_int_t n, const ae_int_t* __restrict x,
+                ae_int_t* __restrict y, ae_state* __restrict _state)
+{
+    const ptrdiff_t unal = ((ptrdiff_t)x) & 31;
+    if( n<=8 )
+    {
+        ae_int_t j;
+        for(j=0; j<=n-1; j++)
+            y[j] = x[j];
+        return;
+    }
+    if(unal == 0)
+    {
+        icopyvx_avx2_xaligned(n, x, y, _state);
+        return;
+    }
+    const ae_int_t offset = 32-unal;
+    memmove(y, x, offset);
+    const ae_int_t nDone = offset / sizeof(ae_int_t);
+    icopyvx_avx2_xaligned(n-nDone, x+nDone, y+nDone, _state);
+}
+
+void rgemv_straight_avx2(const ae_int_t m, const ae_int_t n,
+    const double alpha, const ae_matrix* __restrict a,
+    const double* __restrict x, double* __restrict y, ae_state* _state)
+{
+    ae_int_t i;
+    ae_int_t j;
+    const __m256d* __restrict pX = (const __m256d*)x;
+    const ae_int_t nVec = n >> 2;
+    const ae_int_t nUnroll = nVec >> 3;
+    __m256d sum = _mm256_setzero_pd();
+    for(i=0; i<m; i++) {
+        const __m256d* __restrict pRow = (const __m256d*)a->ptr.pp_double[i];
+        if(nUnroll >= 1) {
+            __m256d u0 = _mm256_mul_pd(pRow[0], pX[0]);
+            __m256d u1 = _mm256_mul_pd(pRow[1], pX[1]);
+            __m256d u2 = _mm256_mul_pd(pRow[2], pX[2]);
+            __m256d u3 = _mm256_mul_pd(pRow[3], pX[3]);
+            __m256d u4 = _mm256_mul_pd(pRow[4], pX[4]);
+            __m256d u5 = _mm256_mul_pd(pRow[5], pX[5]);
+            __m256d u6 = _mm256_mul_pd(pRow[6], pX[6]);
+            __m256d u7 = _mm256_mul_pd(pRow[7], pX[7]);
+            for(j=1; j<nUnroll; j++) {
+              const ae_int_t at = j<<3;
+              u0 = _mm256_add_pd(u0, _mm256_mul_pd(pRow[at], pX[at]));
+              u1 = _mm256_add_pd(u1, _mm256_mul_pd(pRow[at+1], pX[at+1]));
+              u2 = _mm256_add_pd(u2, _mm256_mul_pd(pRow[at+2], pX[at+2]));
+              u3 = _mm256_add_pd(u3, _mm256_mul_pd(pRow[at+3], pX[at+3]));
+              u4 = _mm256_add_pd(u4, _mm256_mul_pd(pRow[at+4], pX[at+4]));
+              u5 = _mm256_add_pd(u5, _mm256_mul_pd(pRow[at+5], pX[at+5]));
+              u6 = _mm256_add_pd(u6, _mm256_mul_pd(pRow[at+6], pX[at+6]));
+              u7 = _mm256_add_pd(u7, _mm256_mul_pd(pRow[at+7], pX[at+7]));
+            }
+            const ae_int_t at = j<<3;
+            switch(nVec-at) {
+            case 7:
+                u6 = _mm256_add_pd(_mm256_mul_pd(pX[at+6], pRow[at+6]), u6);
+            case 6:
+                u5 = _mm256_add_pd(_mm256_mul_pd(pX[at+5], pRow[at+5]), u5);
+            case 5:
+                u4 = _mm256_add_pd(_mm256_mul_pd(pX[at+4], pRow[at+4]), u4);
+            case 4:
+                u3 = _mm256_add_pd(_mm256_mul_pd(pX[at+3], pRow[at+3]), u3);
+            case 3:
+                u2 = _mm256_add_pd(_mm256_mul_pd(pX[at+2], pRow[at+2]), u2);
+            case 2:
+                u1 = _mm256_add_pd(_mm256_mul_pd(pX[at+1], pRow[at+1]), u1);
+            case 1:
+                u0 = _mm256_add_pd(_mm256_mul_pd(pX[at+0], pRow[at+0]), u0);
+            }
+            sum = _mm256_add_pd(
+                _mm256_add_pd(_mm256_add_pd(u0, u1), _mm256_add_pd(u2, u3)),
+                _mm256_add_pd(_mm256_add_pd(u4, u5), _mm256_add_pd(u6, u7)));
+        }
+        else {
+            switch(nVec) {
+            case 0:
+                sum = _mm256_setzero_pd();
+                break;
+            case 1:
+                sum = _mm256_mul_pd(pX[0], pRow[0]);
+                break;
+            case 2:
+                sum = _mm256_add_pd(_mm256_mul_pd(pX[0], pRow[0]), _mm256_mul_pd(pX[1], pRow[1]));
+                break;
+            case 3:
+                sum = _mm256_add_pd(
+                    _mm256_add_pd(_mm256_mul_pd(pX[0], pRow[0]), _mm256_mul_pd(pX[1], pRow[1])),
+                    _mm256_mul_pd(pX[2], pRow[2]));
+                break;
+            case 4:
+                sum = _mm256_add_pd(
+                    _mm256_add_pd(_mm256_mul_pd(pX[0], pRow[0]), _mm256_mul_pd(pX[1], pRow[1])),
+                    _mm256_add_pd(_mm256_mul_pd(pX[2], pRow[2]), _mm256_mul_pd(pX[3], pRow[3])));
+                break;
+            case 5:
+                sum = _mm256_add_pd(
+                    _mm256_add_pd(
+                        _mm256_add_pd(_mm256_mul_pd(pX[0], pRow[0]), _mm256_mul_pd(pX[1], pRow[1])),
+                        _mm256_add_pd(_mm256_mul_pd(pX[2], pRow[2]), _mm256_mul_pd(pX[3], pRow[3]))),
+                    _mm256_mul_pd(pX[4], pRow[4]));
+                break;
+            case 6:
+                sum = _mm256_add_pd(
+                    _mm256_add_pd(
+                        _mm256_add_pd(_mm256_mul_pd(pX[0], pRow[0]), _mm256_mul_pd(pX[1], pRow[1])),
+                        _mm256_add_pd(_mm256_mul_pd(pX[2], pRow[2]), _mm256_mul_pd(pX[3], pRow[3]))),
+                    _mm256_add_pd(_mm256_mul_pd(pX[4], pRow[4]), _mm256_mul_pd(pX[5], pRow[5])));
+                break;
+            case 7:
+                sum = _mm256_add_pd(
+                    _mm256_add_pd(
+                        _mm256_add_pd(_mm256_mul_pd(pX[0], pRow[0]), _mm256_mul_pd(pX[1], pRow[1])),
+                        _mm256_add_pd(_mm256_mul_pd(pX[2], pRow[2]), _mm256_mul_pd(pX[3], pRow[3]))),
+                    _mm256_add_pd(
+                        _mm256_add_pd(_mm256_mul_pd(pX[4], pRow[4]), _mm256_mul_pd(pX[5], pRow[5])),
+                        _mm256_mul_pd(pX[6], pRow[6])));
+                break;
+            }
+        }
+        const __m128d t = _mm_add_pd(_mm256_extractf128_pd(sum, 0), _mm256_extractf128_pd(sum, 1));
+        const double* pComps = (const double*)&t;
+        double ans = pComps[0] + pComps[1];
+        const ae_int_t tail = nVec<<2;
+        for(j=tail; j<n; j++) {
+          ans += a->ptr.pp_double[i][j] * x[j];
+        }
+        y[i] += alpha*ans;
+    }
+}
+
+void rgemv_transposed_avx2(const ae_int_t m, const ae_int_t n,
+    const double alpha, const ae_matrix* __restrict a,
+    const double* __restrict x, double* __restrict y, ae_state* _state)
+{
+    ae_int_t i;
+    ae_int_t j;
+    __m256d* __restrict pY = (__m256d*)y;
+    const ae_int_t nVec = m >> 2;
+
+    for(i=0; i<=n-1; i++)
+    {
+        const __m256d* __restrict pRow = (const __m256d*)a->ptr.pp_double[i];
+        const double v = alpha*x[i];
+        const __m256d vV = _mm256_set1_pd(v);
+        for(j=0; j<nVec; j++)
+        {
+            pY[j] = _mm256_add_pd(_mm256_mul_pd(vV, pRow[j]), pY[j]);
+        }
+        const ae_int_t tail = nVec<<2;
+        for(j=tail; j<m; j++) {
+            y[j] += v*a->ptr.pp_double[i][j];
+        }
+    }
+}
+
+void rgemvx_straight_avx2_xaligned(const ae_int_t m, const ae_int_t n,
+    const double alpha, const ae_matrix* __restrict a, const ae_int_t ia,
+    const ae_int_t ja, const double* __restrict x,
+    double* __restrict y, ae_state* _state)
+{
+    ae_int_t i;
+    ae_int_t j;
+    const __m256d* __restrict pX = (const __m256d*)x;
+    const ae_int_t nVec = n >> 2;
+    const ae_int_t nUnroll = nVec >> 3;
+    __m256d sum = _mm256_setzero_pd();
+    for(i=0; i<m; i++) {
+        const __m256d* __restrict pRow = (const __m256d*)(a->ptr.pp_double[i+ia]+ja);
+        if(nUnroll >= 1) {
+            __m256d u0 = _mm256_mul_pd(ULOAD256PD(pRow[0]), pX[0]);
+            __m256d u1 = _mm256_mul_pd(ULOAD256PD(pRow[1]), pX[1]);
+            __m256d u2 = _mm256_mul_pd(ULOAD256PD(pRow[2]), pX[2]);
+            __m256d u3 = _mm256_mul_pd(ULOAD256PD(pRow[3]), pX[3]);
+            __m256d u4 = _mm256_mul_pd(ULOAD256PD(pRow[4]), pX[4]);
+            __m256d u5 = _mm256_mul_pd(ULOAD256PD(pRow[5]), pX[5]);
+            __m256d u6 = _mm256_mul_pd(ULOAD256PD(pRow[6]), pX[6]);
+            __m256d u7 = _mm256_mul_pd(ULOAD256PD(pRow[7]), pX[7]);
+            for(j=1; j<nUnroll; j++) {
+              const ae_int_t at = j<<3;
+              u0 = _mm256_add_pd(u0, _mm256_mul_pd(ULOAD256PD(pRow[at]), pX[at]));
+              u1 = _mm256_add_pd(u1, _mm256_mul_pd(ULOAD256PD(pRow[at+1]), pX[at+1]));
+              u2 = _mm256_add_pd(u2, _mm256_mul_pd(ULOAD256PD(pRow[at+2]), pX[at+2]));
+              u3 = _mm256_add_pd(u3, _mm256_mul_pd(ULOAD256PD(pRow[at+3]), pX[at+3]));
+              u4 = _mm256_add_pd(u4, _mm256_mul_pd(ULOAD256PD(pRow[at+4]), pX[at+4]));
+              u5 = _mm256_add_pd(u5, _mm256_mul_pd(ULOAD256PD(pRow[at+5]), pX[at+5]));
+              u6 = _mm256_add_pd(u6, _mm256_mul_pd(ULOAD256PD(pRow[at+6]), pX[at+6]));
+              u7 = _mm256_add_pd(u7, _mm256_mul_pd(ULOAD256PD(pRow[at+7]), pX[at+7]));
+            }
+            const ae_int_t at = j<<3;
+            switch(nVec-at) {
+            case 7:
+                u6 = _mm256_add_pd(_mm256_mul_pd(pX[at+6], ULOAD256PD(pRow[at+6])), u6);
+            case 6:
+                u5 = _mm256_add_pd(_mm256_mul_pd(pX[at+5], ULOAD256PD(pRow[at+5])), u5);
+            case 5:
+                u4 = _mm256_add_pd(_mm256_mul_pd(pX[at+4], ULOAD256PD(pRow[at+4])), u4);
+            case 4:
+                u3 = _mm256_add_pd(_mm256_mul_pd(pX[at+3], ULOAD256PD(pRow[at+3])), u3);
+            case 3:
+                u2 = _mm256_add_pd(_mm256_mul_pd(pX[at+2], ULOAD256PD(pRow[at+2])), u2);
+            case 2:
+                u1 = _mm256_add_pd(_mm256_mul_pd(pX[at+1], ULOAD256PD(pRow[at+1])), u1);
+            case 1:
+                u0 = _mm256_add_pd(_mm256_mul_pd(pX[at+0], ULOAD256PD(pRow[at+0])), u0);
+            }
+            sum = _mm256_add_pd(
+                _mm256_add_pd(_mm256_add_pd(u0, u1), _mm256_add_pd(u2, u3)),
+                _mm256_add_pd(_mm256_add_pd(u4, u5), _mm256_add_pd(u6, u7)));
+        }
+        else {
+            switch(nVec) {
+            case 0:
+                sum = _mm256_setzero_pd();
+                break;
+            case 1:
+                sum = _mm256_mul_pd(pX[0], ULOAD256PD(pRow[0]));
+                break;
+            case 2:
+                sum = _mm256_add_pd(
+                    _mm256_mul_pd(pX[0], ULOAD256PD(pRow[0])),
+                    _mm256_mul_pd(pX[1], ULOAD256PD(pRow[1])));
+                break;
+            case 3:
+                sum = _mm256_add_pd(
+                    _mm256_add_pd(
+                        _mm256_mul_pd(pX[0], ULOAD256PD(pRow[0])),
+                        _mm256_mul_pd(pX[1], ULOAD256PD(pRow[1]))),
+                    _mm256_mul_pd(pX[2], ULOAD256PD(pRow[2])));
+                break;
+            case 4:
+                sum = _mm256_add_pd(
+                    _mm256_add_pd(
+                        _mm256_mul_pd(pX[0], ULOAD256PD(pRow[0])),
+                        _mm256_mul_pd(pX[1], ULOAD256PD(pRow[1]))),
+                    _mm256_add_pd(
+                        _mm256_mul_pd(pX[2], ULOAD256PD(pRow[2])),
+                        _mm256_mul_pd(pX[3], ULOAD256PD(pRow[3]))));
+                break;
+            case 5:
+                sum = _mm256_add_pd(
+                    _mm256_add_pd(
+                        _mm256_add_pd(
+                            _mm256_mul_pd(pX[0], ULOAD256PD(pRow[0])),
+                            _mm256_mul_pd(pX[1], ULOAD256PD(pRow[1]))),
+                        _mm256_add_pd(
+                            _mm256_mul_pd(pX[2], ULOAD256PD(pRow[2])),
+                            _mm256_mul_pd(pX[3], ULOAD256PD(pRow[3])))),
+                    _mm256_mul_pd(pX[4], ULOAD256PD(pRow[4])));
+                break;
+            case 6:
+                sum = _mm256_add_pd(
+                    _mm256_add_pd(
+                        _mm256_add_pd(
+                            _mm256_mul_pd(pX[0], ULOAD256PD(pRow[0])),
+                            _mm256_mul_pd(pX[1], ULOAD256PD(pRow[1]))),
+                        _mm256_add_pd(
+                            _mm256_mul_pd(pX[2], ULOAD256PD(pRow[2])),
+                            _mm256_mul_pd(pX[3], ULOAD256PD(pRow[3])))),
+                    _mm256_add_pd(
+                        _mm256_mul_pd(pX[4], ULOAD256PD(pRow[4])),
+                        _mm256_mul_pd(pX[5], ULOAD256PD(pRow[5]))));
+                break;
+            case 7:
+                sum = _mm256_add_pd(
+                    _mm256_add_pd(
+                        _mm256_add_pd(
+                            _mm256_mul_pd(pX[0], ULOAD256PD(pRow[0])),
+                            _mm256_mul_pd(pX[1], ULOAD256PD(pRow[1]))),
+                        _mm256_add_pd(
+                            _mm256_mul_pd(pX[2], ULOAD256PD(pRow[2])),
+                            _mm256_mul_pd(pX[3], ULOAD256PD(pRow[3])))),
+                    _mm256_add_pd(
+                        _mm256_add_pd(
+                            _mm256_mul_pd(pX[4], ULOAD256PD(pRow[4])),
+                            _mm256_mul_pd(pX[5], ULOAD256PD(pRow[5]))),
+                        _mm256_mul_pd(pX[6], ULOAD256PD(pRow[6]))));
+                break;
+            }
+        }
+        const __m128d t = _mm_add_pd(_mm256_extractf128_pd(sum, 0), _mm256_extractf128_pd(sum, 1));
+        const double* pComps = (const double*)&t;
+        double ans = pComps[0] + pComps[1];
+        const ae_int_t tail = nVec<<2;
+        for(j=tail; j<n; j++) {
+          ans += a->ptr.pp_double[i+ia][j+ja] * x[j];
+        }
+        y[i] += alpha*ans;
+    }
+}
+
+void rgemvx_straight_avx2(const ae_int_t m, const ae_int_t n,
+    const double alpha, const ae_matrix* __restrict a, const ae_int_t ia,
+    const ae_int_t ja, const double* __restrict x,
+    double* __restrict y, ae_state* _state)
+{
+    ae_int_t i;
+    ae_int_t j;
+    if( n<=3 ) {
+        for(i=0; i<m; i++) {
+            const double *p_a = a->ptr.pp_double[ia+i]+ja;
+            double v = 0.0;
+            for(j=0; j<n; j++) {
+                v += p_a[j] * x[j];
+            }
+            y[i] += alpha*v;
+        }
+        return;
+    }
+
+    const ptrdiff_t unal = ((ptrdiff_t)x) & 31;
+    if(unal == 0)
+    {
+        rgemvx_straight_avx2_xaligned(m, n, alpha, a, ia, ja, x, y, _state);
+        return;
+    }
+    const ptrdiff_t shift = 4-(unal>>3);
+    for(i=0; i<m; i++) {
+        const double *p_a = a->ptr.pp_double[ia+i]+ja;
+        double v = 0.0;
+        for(j=0; j<shift; j++) {
+            v += p_a[j] * x[j];
+        }
+        y[i] += alpha*v;
+    }
+    rgemvx_straight_avx2_xaligned(m, n-shift, alpha, a, ia, ja+shift, x+shift, y, _state);
+}
+
+void rgemvx_transposed_avx2_yaligned(const ae_int_t m, const ae_int_t n,
+    const double alpha, const ae_matrix* __restrict a, const ae_int_t ia,
+    const ae_int_t ja, const double* __restrict x, double* __restrict y,
+    ae_state* _state)
+{
+    ae_int_t i;
+    ae_int_t j;
+    __m256d* __restrict pY = (__m256d*)y;
+    const ae_int_t nVec = m >> 2;
+
+    for(i=0; i<=n-1; i++)
+    {
+        const __m256d* __restrict pRow = (const __m256d*)(a->ptr.pp_double[i+ia]+ja);
+        const double v = alpha*x[i];
+        const __m256d vV = _mm256_set1_pd(v);
+        for(j=0; j<nVec; j++)
+        {
+            pY[j] = _mm256_add_pd(_mm256_mul_pd(vV, ULOAD256PD(pRow[j])), pY[j]);
+        }
+        const ae_int_t tail = nVec<<2;
+        for(j=tail; j<m; j++) {
+            y[j] += v*a->ptr.pp_double[i+ia][j+ja];
+        }
+    }
+}
+
+void rgemvx_transposed_avx2(const ae_int_t m, const ae_int_t n,
+    const double alpha, const ae_matrix* __restrict a, const ae_int_t ia,
+    const ae_int_t ja, const double* __restrict x, double* __restrict y,
+    ae_state* _state)
+{
+    ae_int_t i;
+    ae_int_t j;
+    if( m<=3 ) {
+        for(i=0; i<n; i++) {
+            const double *p_a = a->ptr.pp_double[ia+i]+ja;
+            const double v = alpha*x[i];
+            for(j=0; j<m; j++) {
+                y[j] += v*p_a[j];
+            }
+        }
+        return;
+    }
+
+    const ptrdiff_t unal = ((ptrdiff_t)y) & 31;
+    if(unal == 0)
+    {
+        rgemvx_transposed_avx2_yaligned(m, n, alpha, a, ia, ja, x, y, _state);
+        return;
+    }
+    const ptrdiff_t shift = 4-(unal>>3);
+    for(i=0; i<n; i++) {
+        const double *p_a = a->ptr.pp_double[ia+i]+ja;
+        const double v = alpha*x[i];
+        for(j=0; j<shift; j++) {
+            y[j] += v*p_a[j];
+        }
+    }
+    rgemvx_transposed_avx2_yaligned(m-shift, n, alpha, a, ia, ja+shift, x, y+shift, _state);
+}
+
+/*************************************************************************
+Block packing function for fast rGEMM. Loads long  WIDTH*LENGTH  submatrix
+with LENGTH<=BLOCK_SIZE and WIDTH<=MICRO_SIZE into contiguous  MICRO_SIZE*
+BLOCK_SIZE row-wise 'horizontal' storage (hence H in the function name).
+
+The matrix occupies first ROUND_LENGTH cols of the  storage  (with  LENGTH
+being rounded up to nearest SIMD granularity).  ROUND_LENGTH  is  returned
+as result. It is guaranteed that ROUND_LENGTH depends only on LENGTH,  and
+that it will be same for all function calls.
+
+Unused rows and columns in [LENGTH,ROUND_LENGTH) range are filled by zeros;
+unused cols in [ROUND_LENGTH,BLOCK_SIZE) range are ignored.
+
+* op=0 means that source is an WIDTH*LENGTH matrix stored with  src_stride
+  stride. The matrix is NOT transposed on load.
+* op=1 means that source is an LENGTH*WIDTH matrix  stored with src_stride
+  that is loaded with transposition
+* present version of the function supports only MICRO_SIZE=2, the behavior
+  is undefined for other micro sizes.
+* the target is properly aligned; the source can be unaligned.
+
+Requires AVX2, does NOT check its presense.
+
+The function is present in two versions, one  with  variable  opsrc_length
+and another one with opsrc_length==block_size==32.
+
+  -- ALGLIB routine --
+     19.07.2021
+     Bochkanov Sergey
+*************************************************************************/
+ae_int_t ablasf_packblkh_avx2(
+    const double *src,
+    ae_int_t src_stride,
+    ae_int_t op,
+    ae_int_t opsrc_length,
+    ae_int_t opsrc_width,
+    double   *dst,
+    ae_int_t block_size,
+    ae_int_t micro_size)
+{
+    ae_int_t i;
+    
+    /*
+     * Write to the storage
+     */
+    if( op==0 )
+    {
+        /*
+         * Copy without transposition
+         */
+        const ae_int_t len8=(opsrc_length>>3)<<3;
+        const double *src1 = src+src_stride;
+        double       *dst1 = dst+block_size;
+        if( opsrc_width==2 )
+        {
+            /*
+             * Width=2
+             */
+            for(i=0; i<len8; i+=8)
+            {
+                _mm256_store_pd(dst+i,    _mm256_loadu_pd(src+i));
+                _mm256_store_pd(dst+i+4,  _mm256_loadu_pd(src+i+4));
+                _mm256_store_pd(dst1+i,   _mm256_loadu_pd(src1+i));
+                _mm256_store_pd(dst1+i+4, _mm256_loadu_pd(src1+i+4));
+            }
+            for(i=len8; i<opsrc_length; i++)
+            {
+                dst[i]  = src[i];
+                dst1[i] = src1[i];
+            }
+        }
+        else
+        {
+            /*
+             * Width=1, pad by zeros
+             */
+            __m256d vz = _mm256_setzero_pd();
+            for(i=0; i<len8; i+=8)
+            {
+                _mm256_store_pd(dst+i,    _mm256_loadu_pd(src+i));
+                _mm256_store_pd(dst+i+4,  _mm256_loadu_pd(src+i+4));
+                _mm256_store_pd(dst1+i,   vz);
+                _mm256_store_pd(dst1+i+4, vz);
+            }
+            for(i=len8; i<opsrc_length; i++)
+            {
+                dst[i]  = src[i];
+                dst1[i] = 0.0;
+            }
+        }
+    }
+    else
+    {
+        /*
+         * Copy with transposition
+         */
+        const ae_int_t stride2 = src_stride<<1;
+        const ae_int_t stride3 = src_stride+stride2;
+        const ae_int_t stride4 = src_stride<<2;
+        const ae_int_t len4=(opsrc_length>>2)<<2;
+        const double *srci = src;
+        double       *dst1 = dst+block_size;
+        if( opsrc_width==2 )
+        {
+            /*
+             * Width=2
+             */
+            for(i=0; i<len4; i+=4)
+            {
+                __m128d s0 = _mm_loadu_pd(srci), s1 = _mm_loadu_pd(srci+src_stride);
+                __m128d s2 = _mm_loadu_pd(srci+stride2), s3 = _mm_loadu_pd(srci+stride3);
+                _mm_store_pd(dst+i,    _mm_unpacklo_pd(s0,s1));
+                _mm_store_pd(dst1+i,   _mm_unpackhi_pd(s0,s1));
+                _mm_store_pd(dst+i+2,  _mm_unpacklo_pd(s2,s3));
+                _mm_store_pd(dst1+i+2, _mm_unpackhi_pd(s2,s3));
+                srci += stride4;
+            }
+            for(i=len4; i<opsrc_length; i++)
+            {
+                dst[i]  = srci[0];
+                dst1[i] = srci[1];
+                srci += src_stride;
+            }
+        }
+        else
+        {
+            /*
+             * Width=1, pad by zeros
+             */
+            __m128d vz = _mm_setzero_pd();
+            for(i=0; i<len4; i+=4)
+            {
+                __m128d s0 = _mm_load_sd(srci), s1 = _mm_load_sd(srci+src_stride);
+                __m128d s2 = _mm_load_sd(srci+stride2), s3 = _mm_load_sd(srci+stride3);
+                _mm_store_pd(dst+i,    _mm_unpacklo_pd(s0,s1));
+                _mm_store_pd(dst+i+2,  _mm_unpacklo_pd(s2,s3));
+                _mm_store_pd(dst1+i,   vz);
+                _mm_store_pd(dst1+i+2, vz);
+                srci += stride4;
+            }
+            for(i=len4; i<opsrc_length; i++)
+            {
+                dst[i]  = srci[0];
+                dst1[i] = 0.0;
+                srci += src_stride;
+            }
+        }
+    }
+    
+    /*
+     * Pad by zeros, if needed
+     */
+    ae_int_t round_length = ((opsrc_length+3)>>2)<<2;
+    for(i=opsrc_length; i<round_length; i++)
+    {
+        dst[i] = 0;
+        dst[i+block_size] = 0;
+    }
+    return round_length;
+}
+
+ae_int_t ablasf_packblkh32_avx2(
+    const double *src,
+    ae_int_t src_stride,
+    ae_int_t op,
+    ae_int_t ignore_opsrc_length,
+    ae_int_t opsrc_width,
+    double   *dst,
+    ae_int_t ignore_block_size,
+    ae_int_t micro_size)
+{
+    ae_int_t i;
+    
+    /*
+     * Write to the storage
+     */
+    if( op==0 )
+    {
+        /*
+         * Copy without transposition
+         */
+        const double *src1 = src+src_stride;
+        double       *dst1 = dst+32;
+        if( opsrc_width==2 )
+        {
+            /*
+             * Width=2
+             */
+            for(i=0; i<32; i+=8)
+            {
+                _mm256_store_pd(dst+i,    _mm256_loadu_pd(src+i));
+                _mm256_store_pd(dst+i+4,  _mm256_loadu_pd(src+i+4));
+                _mm256_store_pd(dst1+i,   _mm256_loadu_pd(src1+i));
+                _mm256_store_pd(dst1+i+4, _mm256_loadu_pd(src1+i+4));
+            }
+        }
+        else
+        {
+            /*
+             * Width=1, pad by zeros
+             */
+            __m256d vz = _mm256_setzero_pd();
+            for(i=0; i<32; i+=8)
+            {
+                _mm256_store_pd(dst+i,    _mm256_loadu_pd(src+i));
+                _mm256_store_pd(dst+i+4,  _mm256_loadu_pd(src+i+4));
+                _mm256_store_pd(dst1+i,   vz);
+                _mm256_store_pd(dst1+i+4, vz);
+            }
+        }
+    }
+    else
+    {
+        /*
+         * Copy with transposition
+         */
+        const ae_int_t stride2 = src_stride<<1;
+        const ae_int_t stride3 = src_stride+stride2;
+        const ae_int_t stride4 = src_stride<<2;
+        const double *srci = src;
+        double       *dst1 = dst+32;
+        if( opsrc_width==2 )
+        {
+            /*
+             * Width=2
+             */
+            for(i=0; i<32; i+=4)
+            {
+                __m128d s0 = _mm_loadu_pd(srci), s1 = _mm_loadu_pd(srci+src_stride);
+                __m128d s2 = _mm_loadu_pd(srci+stride2), s3 = _mm_loadu_pd(srci+stride3);
+                _mm_store_pd(dst+i,    _mm_unpacklo_pd(s0,s1));
+                _mm_store_pd(dst1+i,   _mm_unpackhi_pd(s0,s1));
+                _mm_store_pd(dst+i+2,  _mm_unpacklo_pd(s2,s3));
+                _mm_store_pd(dst1+i+2, _mm_unpackhi_pd(s2,s3));
+                srci += stride4;
+            }
+        }
+        else
+        {
+            /*
+             * Width=1, pad by zeros
+             */
+            __m128d vz = _mm_setzero_pd();
+            for(i=0; i<32; i+=4)
+            {
+                __m128d s0 = _mm_load_sd(srci), s1 = _mm_load_sd(srci+src_stride);
+                __m128d s2 = _mm_load_sd(srci+stride2), s3 = _mm_load_sd(srci+stride3);
+                _mm_store_pd(dst+i,    _mm_unpacklo_pd(s0,s1));
+                _mm_store_pd(dst+i+2,  _mm_unpacklo_pd(s2,s3));
+                _mm_store_pd(dst1+i,   vz);
+                _mm_store_pd(dst1+i+2, vz);
+                srci += stride4;
+            }
+        }
+    }
+    return 32;
+}
+
+/*************************************************************************
+Computes  product   A*transpose(B)  of two MICRO_SIZE*ROUND_LENGTH rowwise 
+'horizontal' matrices, stored with stride=block_size, and writes it to the
+row-wise matrix C.
+
+ROUND_LENGTH is expected to be properly SIMD-rounded length,  as  returned
+by ablasf_packblkh_avx2().
+
+Present version of the function supports only MICRO_SIZE=2,  the  behavior
+is undefined for other micro sizes.
+
+Requires AVX2, does NOT check its presense.
+
+  -- ALGLIB routine --
+     19.07.2021
+     Bochkanov Sergey
+*************************************************************************/
+void ablasf_dotblkh_avx2(
+    const double *src_a,
+    const double *src_b,
+    ae_int_t round_length,
+    ae_int_t block_size,
+    ae_int_t micro_size,
+    double *dst,
+    ae_int_t dst_stride)
+{
+    ae_int_t z;
+    __m256d r00 = _mm256_setzero_pd(), r01 = _mm256_setzero_pd(), r10 = _mm256_setzero_pd(), r11 = _mm256_setzero_pd();
+    if( round_length&0x7 )
+    {
+        /*
+         * round_length is multiple of 4, but not multiple of 8
+         */
+        for(z=0; z<round_length; z+=4, src_a+=4, src_b+=4)
+        {
+            __m256d a0 = _mm256_load_pd(src_a);
+            __m256d a1 = _mm256_load_pd(src_a+block_size);
+            __m256d b0 = _mm256_load_pd(src_b);
+            __m256d b1 = _mm256_load_pd(src_b+block_size);
+            r00 = _mm256_add_pd(_mm256_mul_pd(a0, b0), r00);
+            r01 = _mm256_add_pd(_mm256_mul_pd(a0, b1), r01);
+            r10 = _mm256_add_pd(_mm256_mul_pd(a1, b0), r10);
+            r11 = _mm256_add_pd(_mm256_mul_pd(a1, b1), r11);
+        }
+    }
+    else
+    {
+        /*
+         * round_length is multiple of 8
+         */
+        for(z=0; z<round_length; z+=8, src_a+=8, src_b+=8)
+        {
+            __m256d a0 = _mm256_load_pd(src_a);
+            __m256d a1 = _mm256_load_pd(src_a+block_size);
+            __m256d b0 = _mm256_load_pd(src_b);
+            __m256d b1 = _mm256_load_pd(src_b+block_size);
+            __m256d c0 = _mm256_load_pd(src_a+4);
+            __m256d c1 = _mm256_load_pd(src_a+block_size+4);
+            __m256d d0 = _mm256_load_pd(src_b+4);
+            __m256d d1 = _mm256_load_pd(src_b+block_size+4);
+            r00 = _mm256_add_pd(_mm256_add_pd(r00, _mm256_mul_pd(a0, b0)), _mm256_mul_pd(c0, d0));
+            r01 = _mm256_add_pd(_mm256_add_pd(r01, _mm256_mul_pd(a0, b1)), _mm256_mul_pd(c0, d1));
+            r10 = _mm256_add_pd(_mm256_add_pd(r10, _mm256_mul_pd(a1, b0)), _mm256_mul_pd(c1, d0));
+            r11 = _mm256_add_pd(_mm256_add_pd(r11, _mm256_mul_pd(a1, b1)), _mm256_mul_pd(c1, d1));
+        }
+    }
+    __m256d sum0 = _mm256_hadd_pd(r00,r01);
+    __m256d sum1 = _mm256_hadd_pd(r10,r11);
+    _mm_store_pd(dst,            _mm_add_pd(_mm256_castpd256_pd128(sum0), _mm256_extractf128_pd(sum0,1)));
+    _mm_store_pd(dst+dst_stride, _mm_add_pd(_mm256_castpd256_pd128(sum1), _mm256_extractf128_pd(sum1,1)));
+}
+
+/*************************************************************************
+Y := alpha*X + beta*Y
+
+Requires AVX2, does NOT check its presense.
+
+  -- ALGLIB routine --
+     19.07.2021
+     Bochkanov Sergey
+*************************************************************************/
+void ablasf_daxpby_avx2(
+    ae_int_t     n,
+    double       alpha,
+    const double *src,
+    double       beta,
+    double       *dst)
+{
+    if( beta==1.0 )
+    {
+        /*
+         * The most optimized case: DST := alpha*SRC + DST
+         *
+         * First, we process leading elements with generic C code until DST is aligned.
+         * Then, we process central part, assuming that DST is properly aligned.
+         * Finally, we process tail.
+         */
+        ae_int_t i, n4;
+        __m256d avx_alpha = _mm256_set1_pd(alpha);
+        while( n>0 && (((ptrdiff_t)dst)&31) )
+        {
+            *dst += alpha*(*src);
+            n--;
+            dst++;
+            src++;
+        }
+        n4=(n>>2)<<2;
+        for(i=0; i<n4; i+=4)
+        {
+            __m256d r = _mm256_add_pd(_mm256_mul_pd(avx_alpha, _mm256_loadu_pd(src+i)), _mm256_load_pd(dst+i));
+            _mm256_store_pd(dst+i, r);
+        }
+        for(i=n4; i<n; i++)
+            dst[i] = alpha*src[i]+dst[i];
+    }
+    else if( beta!=0.0 )
+    {
+        /*
+         * Well optimized: DST := alpha*SRC + beta*DST
+         */
+        ae_int_t i, n4;
+        __m256d avx_alpha = _mm256_set1_pd(alpha);
+        __m256d avx_beta  = _mm256_set1_pd(beta);
+        while( n>0 && (((ptrdiff_t)dst)&31) )
+        {
+            *dst = alpha*(*src) + beta*(*dst);
+            n--;
+            dst++;
+            src++;
+        }
+        n4=(n>>2)<<2;
+        for(i=0; i<n4; i+=4)
+        {
+            __m256d r = _mm256_add_pd(
+                _mm256_mul_pd(avx_alpha, _mm256_loadu_pd(src+i)),
+                _mm256_mul_pd(avx_beta,_mm256_load_pd(dst+i)));
+            _mm256_store_pd(dst+i, r);
+        }
+        for(i=n4; i<n; i++)
+            dst[i] = alpha*src[i]+beta*dst[i];
+    }
+    else
+    {
+        /*
+         * Easy case: DST := alpha*SRC
+         */
+        ae_int_t i;
+        for(i=0; i<n; i++)
+            dst[i] = alpha*src[i];
+    }
+}
+
+ae_bool spchol_updatekernelabc4_avx2(double* rowstorage,
+     ae_int_t offss,
+     ae_int_t twidth,
+     ae_int_t offsu,
+     ae_int_t uheight,
+     ae_int_t urank,
+     ae_int_t urowstride,
+     ae_int_t uwidth,
+     const double* diagd,
+     ae_int_t offsd,
+     const ae_int_t* raw2smap,
+     const ae_int_t* superrowidx,
+     ae_int_t urbase,
+     ae_state *_state)
+{
+    ae_int_t k;
+    ae_int_t targetrow;
+    ae_int_t targetcol;
+    
+    /*
+     * Filter out unsupported combinations (ones that are too sparse for the non-SIMD code)
+     */
+    if( twidth<3||twidth>4 )
+    {
+        return ae_false;
+    }
+    if( uwidth<1||uwidth>4 )
+    {
+        return ae_false;
+    }
+    if( urank>4 )
+    {
+        return ae_false;
+    }
+    
+    /*
+     * Shift input arrays to the beginning of the working area.
+     * Prepare SIMD masks
+     */
+    __m256i v_rankmask = _mm256_cmpgt_epi64(_mm256_set_epi64x(urank, urank, urank, urank), _mm256_set_epi64x(3, 2, 1, 0));
+    double *update_storage = rowstorage+offsu;
+    double *target_storage = rowstorage+offss;
+    superrowidx += urbase;
+    
+    /*
+     * Load head of the update matrix
+     */
+    __m256d v_d0123 = _mm256_maskload_pd(diagd+offsd, v_rankmask);
+    __m256d u_0_0123 = _mm256_setzero_pd();
+    __m256d u_1_0123 = _mm256_setzero_pd();
+    __m256d u_2_0123 = _mm256_setzero_pd();
+    __m256d u_3_0123 = _mm256_setzero_pd();
+    for(k=0; k<=uwidth-1; k++)
+    {
+        targetcol = raw2smap[superrowidx[k]];
+        if( targetcol==0 )
+            u_0_0123 = _mm256_mul_pd(v_d0123, _mm256_maskload_pd(update_storage+k*urowstride, v_rankmask));
+        if( targetcol==1 )
+            u_1_0123 = _mm256_mul_pd(v_d0123, _mm256_maskload_pd(update_storage+k*urowstride, v_rankmask));
+        if( targetcol==2 )
+            u_2_0123 = _mm256_mul_pd(v_d0123, _mm256_maskload_pd(update_storage+k*urowstride, v_rankmask));
+        if( targetcol==3 )
+            u_3_0123 = _mm256_mul_pd(v_d0123, _mm256_maskload_pd(update_storage+k*urowstride, v_rankmask));
+    }
+    
+    /*
+     * Transpose head
+     */
+    __m256d u01_lo = _mm256_unpacklo_pd(u_0_0123,u_1_0123);
+    __m256d u01_hi = _mm256_unpackhi_pd(u_0_0123,u_1_0123);
+    __m256d u23_lo = _mm256_unpacklo_pd(u_2_0123,u_3_0123);
+    __m256d u23_hi = _mm256_unpackhi_pd(u_2_0123,u_3_0123);
+    __m256d u_0123_0 = _mm256_permute2f128_pd(u01_lo, u23_lo, 0x20);
+    __m256d u_0123_1 = _mm256_permute2f128_pd(u01_hi, u23_hi, 0x20);
+    __m256d u_0123_2 = _mm256_permute2f128_pd(u23_lo, u01_lo, 0x13);
+    __m256d u_0123_3 = _mm256_permute2f128_pd(u23_hi, u01_hi, 0x13);
+    
+    /*
+     * Run update
+     */
+    if( urank==1 )
+    {
+        for(k=0; k<=uheight-1; k++)
+        {
+            targetrow = raw2smap[superrowidx[k]]*4;
+            double *update_row = rowstorage+offsu+k*urowstride;
+            _mm256_store_pd(target_storage+targetrow,
+                _mm256_sub_pd(_mm256_load_pd(target_storage+targetrow),
+                    _mm256_mul_pd(_mm256_broadcast_sd(update_row+0), u_0123_0)));
+        }
+    }
+    if( urank==2 )
+    {
+        for(k=0; k<=uheight-1; k++)
+        {
+            targetrow = raw2smap[superrowidx[k]]*4;
+            double *update_row = rowstorage+offsu+k*urowstride;
+            _mm256_store_pd(target_storage+targetrow,
+                _mm256_sub_pd(_mm256_sub_pd(_mm256_load_pd(target_storage+targetrow),
+                    _mm256_mul_pd(_mm256_broadcast_sd(update_row+1), u_0123_1)),
+                    _mm256_mul_pd(_mm256_broadcast_sd(update_row+0), u_0123_0)));
+        }
+    }
+    if( urank==3 )
+    {
+        for(k=0; k<=uheight-1; k++)
+        {
+            targetrow = raw2smap[superrowidx[k]]*4;
+            double *update_row = rowstorage+offsu+k*urowstride;
+            _mm256_store_pd(target_storage+targetrow,
+                _mm256_sub_pd(_mm256_sub_pd(_mm256_sub_pd(_mm256_load_pd(target_storage+targetrow),
+                    _mm256_mul_pd(_mm256_broadcast_sd(update_row+2), u_0123_2)),
+                    _mm256_mul_pd(_mm256_broadcast_sd(update_row+1), u_0123_1)),
+                    _mm256_mul_pd(_mm256_broadcast_sd(update_row+0), u_0123_0)));
+        }
+    }
+    if( urank==4 )
+    {
+        for(k=0; k<=uheight-1; k++)
+        {
+            targetrow = raw2smap[superrowidx[k]]*4;
+            double *update_row = rowstorage+offsu+k*urowstride;
+            _mm256_store_pd(target_storage+targetrow,
+                _mm256_sub_pd(_mm256_sub_pd(_mm256_sub_pd(_mm256_sub_pd(_mm256_load_pd(target_storage+targetrow),
+                    _mm256_mul_pd(_mm256_broadcast_sd(update_row+3), u_0123_3)),
+                    _mm256_mul_pd(_mm256_broadcast_sd(update_row+2), u_0123_2)),
+                    _mm256_mul_pd(_mm256_broadcast_sd(update_row+1), u_0123_1)),
+                    _mm256_mul_pd(_mm256_broadcast_sd(update_row+0), u_0123_0)));
+        }
+    }
+    return ae_true;
+}
+
+ae_bool spchol_updatekernel4444_avx2(
+     double*  rowstorage,
+     ae_int_t offss,
+     ae_int_t sheight,
+     ae_int_t offsu,
+     ae_int_t uheight,
+     const double*  diagd,
+     ae_int_t offsd,
+     const ae_int_t* raw2smap,
+     const ae_int_t* superrowidx,
+     ae_int_t urbase,
+     ae_state *_state)
+{
+    ae_int_t k;
+    ae_int_t targetrow;
+    ae_int_t offsk;
+    __m256d v_negd_u0, v_negd_u1, v_negd_u2, v_negd_u3, v_negd;
+    __m256d v_w0, v_w1, v_w2, v_w3, u01_lo, u01_hi, u23_lo, u23_hi;
+    
+    /*
+     * Compute W = -D*transpose(U[0:3])
+     */
+    v_negd = _mm256_mul_pd(_mm256_loadu_pd(diagd+offsd),_mm256_set1_pd(-1.0));
+    v_negd_u0   = _mm256_mul_pd(_mm256_load_pd(rowstorage+offsu+0*4),v_negd);
+    v_negd_u1   = _mm256_mul_pd(_mm256_load_pd(rowstorage+offsu+1*4),v_negd);
+    v_negd_u2   = _mm256_mul_pd(_mm256_load_pd(rowstorage+offsu+2*4),v_negd);
+    v_negd_u3   = _mm256_mul_pd(_mm256_load_pd(rowstorage+offsu+3*4),v_negd);
+    u01_lo = _mm256_unpacklo_pd(v_negd_u0,v_negd_u1);
+    u01_hi = _mm256_unpackhi_pd(v_negd_u0,v_negd_u1);
+    u23_lo = _mm256_unpacklo_pd(v_negd_u2,v_negd_u3);
+    u23_hi = _mm256_unpackhi_pd(v_negd_u2,v_negd_u3);
+    v_w0 = _mm256_permute2f128_pd(u01_lo, u23_lo, 0x20);
+    v_w1 = _mm256_permute2f128_pd(u01_hi, u23_hi, 0x20);
+    v_w2 = _mm256_permute2f128_pd(u23_lo, u01_lo, 0x13);
+    v_w3 = _mm256_permute2f128_pd(u23_hi, u01_hi, 0x13);
+    
+    //
+    // Compute update S:= S + row_scatter(U*W)
+    //
+    if( sheight==uheight )
+    {
+        /*
+         * No row scatter, the most efficient code
+         */
+        for(k=0; k<=uheight-1; k++)
+        {
+            __m256d target;
+            
+            targetrow = offss+k*4;
+            offsk = offsu+k*4;
+            
+            target = _mm256_load_pd(rowstorage+targetrow);
+            target = _mm256_add_pd(_mm256_mul_pd(_mm256_broadcast_sd(rowstorage+offsk+0),v_w0),target);
+            target = _mm256_add_pd(_mm256_mul_pd(_mm256_broadcast_sd(rowstorage+offsk+1),v_w1),target);
+            target = _mm256_add_pd(_mm256_mul_pd(_mm256_broadcast_sd(rowstorage+offsk+2),v_w2),target);
+            target = _mm256_add_pd(_mm256_mul_pd(_mm256_broadcast_sd(rowstorage+offsk+3),v_w3),target);
+            _mm256_store_pd(rowstorage+targetrow, target);
+        }
+    }
+    else
+    {
+        /*
+         * Row scatter is performed, less efficient code using double mapping to determine target row index
+         */
+        for(k=0; k<=uheight-1; k++)
+        {
+            __m256d v_uk0, v_uk1, v_uk2, v_uk3, target;
+            
+            targetrow = offss+raw2smap[superrowidx[urbase+k]]*4;
+            offsk = offsu+k*4;
+            
+            target = _mm256_load_pd(rowstorage+targetrow);
+            v_uk0 = _mm256_broadcast_sd(rowstorage+offsk+0);
+            v_uk1 = _mm256_broadcast_sd(rowstorage+offsk+1);
+            v_uk2 = _mm256_broadcast_sd(rowstorage+offsk+2);
+            v_uk3 = _mm256_broadcast_sd(rowstorage+offsk+3);
+            target = _mm256_add_pd(_mm256_mul_pd(v_uk0,v_w0),target);
+            target = _mm256_add_pd(_mm256_mul_pd(v_uk1,v_w1),target);
+            target = _mm256_add_pd(_mm256_mul_pd(v_uk2,v_w2),target);
+            target = _mm256_add_pd(_mm256_mul_pd(v_uk3,v_w3),target);
+            _mm256_store_pd(rowstorage+targetrow, target);
+        }
+    }
+    return ae_true;
+}
+
+
+/*************************************************************************
+Fast kernel for biharmonic panel with NY=1
+
+INPUT PARAMETERS:
+    D0, D1, D2      -   evaluation point minus (Panel.C0,Panel.C1,Panel.C2)
+
+OUTPUT PARAMETERS:
+    F               -   model value
+    InvPowRPPlus1   -   1/(R^(P+1))
+
+  -- ALGLIB --
+     Copyright 26.08.2022 by Sergey Bochkanov
+*************************************************************************/
+ae_bool rbfv3farfields_bhpaneleval1fastkernel16_avx2(double d0,
+     double d1,
+     double d2,
+     const double* pnma,
+     const double* pnmb,
+     const double* pmmcdiag,
+     const double* ynma,
+     const double* tblrmodmn,
+     double* f,
+     double* invpowrpplus1,
+     ae_state *_state)
+{
+    ae_int_t n;
+    double r, r2, r01, invr;
+    double sintheta, costheta;
+    ae_complex expiphi, expiphi2, expiphi3, expiphi4;
+    ae_int_t jj;
+    ae_bool result;
+
+    *f = 0.0;
+    *invpowrpplus1 = 0.0;
+    result = ae_true;
+    
+    /*
+     *Convert to spherical polar coordinates.
+     *
+     * NOTE: we make sure that R is non-zero by adding extremely small perturbation
+     */
+    r2 = d0*d0+d1*d1+d2*d2+ae_minrealnumber;
+    r = ae_sqrt(r2, _state);
+    r01 = ae_sqrt(d0*d0+d1*d1+ae_minrealnumber, _state);
+    costheta = d2/r;
+    sintheta = r01/r;
+    expiphi.x = d0/r01;
+    expiphi.y = d1/r01;
+    invr = (double)1/r;
+    
+    /*
+     * prepare precomputed quantities
+     */
+    double powsintheta2 = sintheta*sintheta;
+    double powsintheta3 = powsintheta2*sintheta;
+    double powsintheta4 = powsintheta2*powsintheta2;
+    expiphi2.x = expiphi.x*expiphi.x-expiphi.y*expiphi.y;
+    expiphi2.y = 2*expiphi.x*expiphi.y;
+    expiphi3.x = expiphi2.x*expiphi.x-expiphi2.y*expiphi.y;
+    expiphi3.y = expiphi2.x*expiphi.y+expiphi.x*expiphi2.y;
+    expiphi4.x = expiphi2.x*expiphi2.x-expiphi2.y*expiphi2.y;
+    expiphi4.y = 2*expiphi2.x*expiphi2.y;
+    
+    /*
+     * Compute far field expansion for a cluster of basis functions f=r
+     *
+     * NOTE: the original paper by Beatson et al. uses f=r as the basis function,
+     *       whilst ALGLIB uses f=-r due to conditional positive definiteness requirement.
+     *       We will perform conversion later.
+     */
+    __m256d v_costheta      = _mm256_set1_pd(costheta);
+    __m256d v_r2            = _mm256_set1_pd(r2);
+    __m256d v_f             = _mm256_setzero_pd();
+    __m256d v_invr          = _mm256_set1_pd(invr);
+    __m256d v_powsinthetaj  = _mm256_set_pd(powsintheta3, powsintheta2, sintheta, 1.0);
+    __m256d v_powsintheta4  = _mm256_set1_pd(powsintheta4);
+    __m256d v_expijphix     = _mm256_set_pd(expiphi3.x, expiphi2.x, expiphi.x, 1.0);
+    __m256d v_expijphiy     = _mm256_set_pd(expiphi3.y, expiphi2.y, expiphi.y, 0.0);
+    __m256d v_expi4phix     = _mm256_set1_pd(expiphi4.x);
+    __m256d v_expi4phiy     = _mm256_set1_pd(expiphi4.y);
+    *f = (double)(0);
+    for(jj=0; jj<4; jj++)
+    {
+        __m256d pnm_cur         = _mm256_setzero_pd(), pnm_prev = _mm256_setzero_pd(), pnm_new;
+        __m256d v_powrminusj1   = _mm256_set1_pd(invr);
+        for(n=0; n<jj*4; n++)
+            v_powrminusj1 = _mm256_mul_pd(v_powrminusj1, v_invr);
+        for(n=jj*4; n<16; n++)
+        {
+            ae_int_t j0=jj*4;
+            
+            
+            pnm_new = _mm256_mul_pd(v_powsinthetaj, _mm256_load_pd(pmmcdiag+n*16+j0));
+            pnm_new = _mm256_add_pd(pnm_new, _mm256_mul_pd(v_costheta,_mm256_mul_pd(pnm_cur,_mm256_load_pd(pnma+n*16+j0))));
+            pnm_new = _mm256_add_pd(pnm_new, _mm256_mul_pd(pnm_prev,_mm256_load_pd(pnmb+n*16+j0)));
+            pnm_prev = pnm_cur;
+            pnm_cur  = pnm_new;
+            
+            __m256d v_tmp = _mm256_mul_pd(pnm_cur, _mm256_load_pd(ynma+n*16+j0));
+            __m256d v_sphericalx = _mm256_mul_pd(v_tmp, v_expijphix);
+            __m256d v_sphericaly = _mm256_mul_pd(v_tmp, v_expijphiy);
+            
+            __m256d v_summnx = _mm256_add_pd(_mm256_mul_pd(v_r2,_mm256_load_pd(tblrmodmn+n*64+j0+32)),_mm256_load_pd(tblrmodmn+n*64+j0));
+            __m256d v_summny = _mm256_add_pd(_mm256_mul_pd(v_r2,_mm256_load_pd(tblrmodmn+n*64+j0+48)),_mm256_load_pd(tblrmodmn+n*64+j0+16));
+            
+            __m256d v_z = _mm256_sub_pd(_mm256_mul_pd(v_sphericalx,v_summnx),_mm256_mul_pd(v_sphericaly,v_summny));
+            
+            v_f = _mm256_add_pd(v_f, _mm256_mul_pd(v_powrminusj1, v_z));
+            v_powrminusj1 = _mm256_mul_pd(v_powrminusj1, v_invr);
+        }
+        __m256d v_expijphix_new = _mm256_sub_pd(_mm256_mul_pd(v_expijphix,v_expi4phix),_mm256_mul_pd(v_expijphiy,v_expi4phiy));
+        __m256d v_expijphiy_new = _mm256_add_pd(_mm256_mul_pd(v_expijphix,v_expi4phiy),_mm256_mul_pd(v_expijphiy,v_expi4phix));
+        v_powsinthetaj = _mm256_mul_pd(v_powsinthetaj, v_powsintheta4);
+        v_expijphix = v_expijphix_new;
+        v_expijphiy = v_expijphiy_new;
+    }
+            
+    double ttt[4];
+    _mm256_storeu_pd(ttt, v_f);
+    for(int k=0; k<4; k++)
+        *f += ttt[k];
+    
+    double r4 = r2*r2;
+    double r8 = r4*r4;
+    double r16 = r8*r8;
+    *invpowrpplus1 = 1/r16;
+    
+    return result;
+}
+
+
+/*************************************************************************
+Fast kernel for biharmonic panel with general NY
+
+INPUT PARAMETERS:
+    D0, D1, D2      -   evaluation point minus (Panel.C0,Panel.C1,Panel.C2)
+
+OUTPUT PARAMETERS:
+    F               -   array[NY], model value
+    InvPowRPPlus1   -   1/(R^(P+1))
+
+  -- ALGLIB --
+     Copyright 26.08.2022 by Sergey Bochkanov
+*************************************************************************/
+ae_bool rbfv3farfields_bhpanelevalfastkernel16_avx2(double d0,
+     double d1,
+     double d2,
+     ae_int_t ny,
+     const double* pnma,
+     const double* pnmb,
+     const double* pmmcdiag,
+     const double* ynma,
+     const double* tblrmodmn,
+     double* f,
+     double* invpowrpplus1,
+     ae_state *_state)
+{
+    ae_int_t n;
+    double r, r2, r01, invr;
+    double sintheta, costheta;
+    ae_complex expiphi, expiphi2, expiphi3, expiphi4;
+    ae_int_t jj;
+    
+    /*
+     * Precomputed buffer which is enough for NY up to 16
+     */
+    __m256d v_f[16];
+    if( ny>16 )
+        return ae_false;
+    for(int k=0; k<ny; k++)
+    {
+        v_f[k] = _mm256_setzero_pd();
+        f[k] = 0.0;
+    }
+    
+    /*
+     *Convert to spherical polar coordinates.
+     *
+     * NOTE: we make sure that R is non-zero by adding extremely small perturbation
+     */
+    r2 = d0*d0+d1*d1+d2*d2+ae_minrealnumber;
+    r = ae_sqrt(r2, _state);
+    r01 = ae_sqrt(d0*d0+d1*d1+ae_minrealnumber, _state);
+    costheta = d2/r;
+    sintheta = r01/r;
+    expiphi.x = d0/r01;
+    expiphi.y = d1/r01;
+    invr = (double)1/r;
+    
+    /*
+     * prepare precomputed quantities
+     */
+    double powsintheta2 = sintheta*sintheta;
+    double powsintheta3 = powsintheta2*sintheta;
+    double powsintheta4 = powsintheta2*powsintheta2;
+    expiphi2.x = expiphi.x*expiphi.x-expiphi.y*expiphi.y;
+    expiphi2.y = 2*expiphi.x*expiphi.y;
+    expiphi3.x = expiphi2.x*expiphi.x-expiphi2.y*expiphi.y;
+    expiphi3.y = expiphi2.x*expiphi.y+expiphi.x*expiphi2.y;
+    expiphi4.x = expiphi2.x*expiphi2.x-expiphi2.y*expiphi2.y;
+    expiphi4.y = 2*expiphi2.x*expiphi2.y;
+    
+    /*
+     * Compute far field expansion for a cluster of basis functions f=r
+     *
+     * NOTE: the original paper by Beatson et al. uses f=r as the basis function,
+     *       whilst ALGLIB uses f=-r due to conditional positive definiteness requirement.
+     *       We will perform conversion later.
+     */
+    __m256d v_costheta      = _mm256_set1_pd(costheta);
+    __m256d v_r2            = _mm256_set1_pd(r2);
+    __m256d v_invr          = _mm256_set1_pd(invr);
+    __m256d v_powsinthetaj  = _mm256_set_pd(powsintheta3, powsintheta2, sintheta, 1.0);
+    __m256d v_powsintheta4  = _mm256_set1_pd(powsintheta4);
+    __m256d v_expijphix     = _mm256_set_pd(expiphi3.x, expiphi2.x, expiphi.x, 1.0);
+    __m256d v_expijphiy     = _mm256_set_pd(expiphi3.y, expiphi2.y, expiphi.y, 0.0);
+    __m256d v_expi4phix     = _mm256_set1_pd(expiphi4.x);
+    __m256d v_expi4phiy     = _mm256_set1_pd(expiphi4.y);
+    *f = (double)(0);
+    for(jj=0; jj<4; jj++)
+    {
+        __m256d pnm_cur         = _mm256_setzero_pd(), pnm_prev = _mm256_setzero_pd(), pnm_new;
+        __m256d v_powrminusj1   = _mm256_set1_pd(invr);
+        for(n=0; n<jj*4; n++)
+            v_powrminusj1 = _mm256_mul_pd(v_powrminusj1, v_invr);
+        for(n=jj*4; n<16; n++)
+        {
+            ae_int_t j0=jj*4;
+            
+            pnm_new = _mm256_mul_pd(v_powsinthetaj, _mm256_load_pd(pmmcdiag+n*16+j0));
+            pnm_new = _mm256_add_pd(pnm_new, _mm256_mul_pd(v_costheta,_mm256_mul_pd(pnm_cur,_mm256_load_pd(pnma+n*16+j0))));
+            pnm_new = _mm256_add_pd(pnm_new, _mm256_mul_pd(pnm_prev,_mm256_load_pd(pnmb+n*16+j0)));
+            pnm_prev = pnm_cur;
+            pnm_cur  = pnm_new;
+            
+            __m256d v_tmp = _mm256_mul_pd(pnm_cur, _mm256_load_pd(ynma+n*16+j0));
+            __m256d v_sphericalx = _mm256_mul_pd(v_tmp, v_expijphix);
+            __m256d v_sphericaly = _mm256_mul_pd(v_tmp, v_expijphiy);
+            
+            const double *p_rmodmn = tblrmodmn+n*64+j0;
+            for(int k=0; k<ny; k++)
+            {
+                __m256d v_summnx = _mm256_add_pd(_mm256_mul_pd(v_r2,_mm256_load_pd(p_rmodmn+32)),_mm256_load_pd(p_rmodmn));
+                __m256d v_summny = _mm256_add_pd(_mm256_mul_pd(v_r2,_mm256_load_pd(p_rmodmn+48)),_mm256_load_pd(p_rmodmn+16));
+                __m256d v_z = _mm256_sub_pd(_mm256_mul_pd(v_sphericalx,v_summnx),_mm256_mul_pd(v_sphericaly,v_summny));
+                v_f[k] = _mm256_add_pd(v_f[k], _mm256_mul_pd(v_powrminusj1, v_z));
+                p_rmodmn += 1024;
+            }
+            v_powrminusj1 = _mm256_mul_pd(v_powrminusj1, v_invr);
+        }
+        __m256d v_expijphix_new = _mm256_sub_pd(_mm256_mul_pd(v_expijphix,v_expi4phix),_mm256_mul_pd(v_expijphiy,v_expi4phiy));
+        __m256d v_expijphiy_new = _mm256_add_pd(_mm256_mul_pd(v_expijphix,v_expi4phiy),_mm256_mul_pd(v_expijphiy,v_expi4phix));
+        v_powsinthetaj = _mm256_mul_pd(v_powsinthetaj, v_powsintheta4);
+        v_expijphix = v_expijphix_new;
+        v_expijphiy = v_expijphiy_new;
+    }
+            
+for(int t=0; t<ny; t++)
+{
+    double ttt[4];
+    _mm256_storeu_pd(ttt, v_f[t]);
+    for(int k=0; k<4; k++)
+        f[t] += ttt[k];
+}
+    
+    double r4 = r2*r2;
+    double r8 = r4*r4;
+    double r16 = r8*r8;
+    *invpowrpplus1 = 1/r16;
+    
+    return ae_true;
+}
+
+/* ALGLIB_NO_FAST_KERNELS, _ALGLIB_HAS_AVX2_INTRINSICS */
+#endif
+
+
+}
+