rbnacl-libsodium 1.0.11 → 1.0.13

data/vendor/libsodium/src/libsodium/crypto_stream/salsa20/xmm6int/salsa20_xmm6int-sse2.h

@@ -0,0 +1,8 @@
+
+#include <stdint.h>
+
+#include "../stream_salsa20.h"
+#include "crypto_stream_salsa20.h"
+
+extern struct crypto_stream_salsa20_implementation
+    crypto_stream_salsa20_xmm6int_sse2_implementation;

data/vendor/libsodium/src/libsodium/crypto_stream/salsa20/xmm6int/u0.h

@@ -0,0 +1,195 @@
+if (bytes > 0) {
+    __m128i diag0 = _mm_loadu_si128((__m128i *) (x + 0));
+    __m128i diag1 = _mm_loadu_si128((__m128i *) (x + 4));
+    __m128i diag2 = _mm_loadu_si128((__m128i *) (x + 8));
+    __m128i diag3 = _mm_loadu_si128((__m128i *) (x + 12));
+    __m128i a0, a1, a2, a3, a4, a5, a6, a7;
+    __m128i b0, b1, b2, b3, b4, b5, b6, b7;
+    uint8_t partialblock[64];
+
+    int     i;
+
+    a0 = diag1;
+    for (i = 0; i < ROUNDS; i += 4) {
+        a0    = _mm_add_epi32(a0, diag0);
+        a1    = diag0;
+        b0    = a0;
+        a0    = _mm_slli_epi32(a0, 7);
+        b0    = _mm_srli_epi32(b0, 25);
+        diag3 = _mm_xor_si128(diag3, a0);
+
+        diag3 = _mm_xor_si128(diag3, b0);
+
+        a1    = _mm_add_epi32(a1, diag3);
+        a2    = diag3;
+        b1    = a1;
+        a1    = _mm_slli_epi32(a1, 9);
+        b1    = _mm_srli_epi32(b1, 23);
+        diag2 = _mm_xor_si128(diag2, a1);
+        diag3 = _mm_shuffle_epi32(diag3, 0x93);
+        diag2 = _mm_xor_si128(diag2, b1);
+
+        a2    = _mm_add_epi32(a2, diag2);
+        a3    = diag2;
+        b2    = a2;
+        a2    = _mm_slli_epi32(a2, 13);
+        b2    = _mm_srli_epi32(b2, 19);
+        diag1 = _mm_xor_si128(diag1, a2);
+        diag2 = _mm_shuffle_epi32(diag2, 0x4e);
+        diag1 = _mm_xor_si128(diag1, b2);
+
+        a3    = _mm_add_epi32(a3, diag1);
+        a4    = diag3;
+        b3    = a3;
+        a3    = _mm_slli_epi32(a3, 18);
+        b3    = _mm_srli_epi32(b3, 14);
+        diag0 = _mm_xor_si128(diag0, a3);
+        diag1 = _mm_shuffle_epi32(diag1, 0x39);
+        diag0 = _mm_xor_si128(diag0, b3);
+
+        a4    = _mm_add_epi32(a4, diag0);
+        a5    = diag0;
+        b4    = a4;
+        a4    = _mm_slli_epi32(a4, 7);
+        b4    = _mm_srli_epi32(b4, 25);
+        diag1 = _mm_xor_si128(diag1, a4);
+
+        diag1 = _mm_xor_si128(diag1, b4);
+
+        a5    = _mm_add_epi32(a5, diag1);
+        a6    = diag1;
+        b5    = a5;
+        a5    = _mm_slli_epi32(a5, 9);
+        b5    = _mm_srli_epi32(b5, 23);
+        diag2 = _mm_xor_si128(diag2, a5);
+        diag1 = _mm_shuffle_epi32(diag1, 0x93);
+        diag2 = _mm_xor_si128(diag2, b5);
+
+        a6    = _mm_add_epi32(a6, diag2);
+        a7    = diag2;
+        b6    = a6;
+        a6    = _mm_slli_epi32(a6, 13);
+        b6    = _mm_srli_epi32(b6, 19);
+        diag3 = _mm_xor_si128(diag3, a6);
+        diag2 = _mm_shuffle_epi32(diag2, 0x4e);
+        diag3 = _mm_xor_si128(diag3, b6);
+
+        a7    = _mm_add_epi32(a7, diag3);
+        a0    = diag1;
+        b7    = a7;
+        a7    = _mm_slli_epi32(a7, 18);
+        b7    = _mm_srli_epi32(b7, 14);
+        diag0 = _mm_xor_si128(diag0, a7);
+        diag3 = _mm_shuffle_epi32(diag3, 0x39);
+        diag0 = _mm_xor_si128(diag0, b7);
+
+        a0    = _mm_add_epi32(a0, diag0);
+        a1    = diag0;
+        b0    = a0;
+        a0    = _mm_slli_epi32(a0, 7);
+        b0    = _mm_srli_epi32(b0, 25);
+        diag3 = _mm_xor_si128(diag3, a0);
+
+        diag3 = _mm_xor_si128(diag3, b0);
+
+        a1    = _mm_add_epi32(a1, diag3);
+        a2    = diag3;
+        b1    = a1;
+        a1    = _mm_slli_epi32(a1, 9);
+        b1    = _mm_srli_epi32(b1, 23);
+        diag2 = _mm_xor_si128(diag2, a1);
+        diag3 = _mm_shuffle_epi32(diag3, 0x93);
+        diag2 = _mm_xor_si128(diag2, b1);
+
+        a2    = _mm_add_epi32(a2, diag2);
+        a3    = diag2;
+        b2    = a2;
+        a2    = _mm_slli_epi32(a2, 13);
+        b2    = _mm_srli_epi32(b2, 19);
+        diag1 = _mm_xor_si128(diag1, a2);
+        diag2 = _mm_shuffle_epi32(diag2, 0x4e);
+        diag1 = _mm_xor_si128(diag1, b2);
+
+        a3    = _mm_add_epi32(a3, diag1);
+        a4    = diag3;
+        b3    = a3;
+        a3    = _mm_slli_epi32(a3, 18);
+        b3    = _mm_srli_epi32(b3, 14);
+        diag0 = _mm_xor_si128(diag0, a3);
+        diag1 = _mm_shuffle_epi32(diag1, 0x39);
+        diag0 = _mm_xor_si128(diag0, b3);
+
+        a4    = _mm_add_epi32(a4, diag0);
+        a5    = diag0;
+        b4    = a4;
+        a4    = _mm_slli_epi32(a4, 7);
+        b4    = _mm_srli_epi32(b4, 25);
+        diag1 = _mm_xor_si128(diag1, a4);
+
+        diag1 = _mm_xor_si128(diag1, b4);
+
+        a5    = _mm_add_epi32(a5, diag1);
+        a6    = diag1;
+        b5    = a5;
+        a5    = _mm_slli_epi32(a5, 9);
+        b5    = _mm_srli_epi32(b5, 23);
+        diag2 = _mm_xor_si128(diag2, a5);
+        diag1 = _mm_shuffle_epi32(diag1, 0x93);
+        diag2 = _mm_xor_si128(diag2, b5);
+
+        a6    = _mm_add_epi32(a6, diag2);
+        a7    = diag2;
+        b6    = a6;
+        a6    = _mm_slli_epi32(a6, 13);
+        b6    = _mm_srli_epi32(b6, 19);
+        diag3 = _mm_xor_si128(diag3, a6);
+        diag2 = _mm_shuffle_epi32(diag2, 0x4e);
+        diag3 = _mm_xor_si128(diag3, b6);
+
+        a7    = _mm_add_epi32(a7, diag3);
+        a0    = diag1;
+        b7    = a7;
+        a7    = _mm_slli_epi32(a7, 18);
+        b7    = _mm_srli_epi32(b7, 14);
+        diag0 = _mm_xor_si128(diag0, a7);
+        diag3 = _mm_shuffle_epi32(diag3, 0x39);
+        diag0 = _mm_xor_si128(diag0, b7);
+    }
+
+    diag0 = _mm_add_epi32(diag0, _mm_loadu_si128((__m128i *) (x + 0)));
+    diag1 = _mm_add_epi32(diag1, _mm_loadu_si128((__m128i *) (x + 4)));
+    diag2 = _mm_add_epi32(diag2, _mm_loadu_si128((__m128i *) (x + 8)));
+    diag3 = _mm_add_epi32(diag3, _mm_loadu_si128((__m128i *) (x + 12)));
+
+#define ONEQUAD_SHUFFLE(A, B, C, D)                      \
+    do {                                                 \
+        uint32_t in##A = _mm_cvtsi128_si32(diag0);       \
+        uint32_t in##B = _mm_cvtsi128_si32(diag1);       \
+        uint32_t in##C = _mm_cvtsi128_si32(diag2);       \
+        uint32_t in##D = _mm_cvtsi128_si32(diag3);       \
+        diag0          = _mm_shuffle_epi32(diag0, 0x39); \
+        diag1          = _mm_shuffle_epi32(diag1, 0x39); \
+        diag2          = _mm_shuffle_epi32(diag2, 0x39); \
+        diag3          = _mm_shuffle_epi32(diag3, 0x39); \
+        *(uint32_t *) (partialblock + (A * 4)) = in##A;  \
+        *(uint32_t *) (partialblock + (B * 4)) = in##B;  \
+        *(uint32_t *) (partialblock + (C * 4)) = in##C;  \
+        *(uint32_t *) (partialblock + (D * 4)) = in##D;  \
+    } while (0)
+
+#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D)
+
+    ONEQUAD(0, 12, 8, 4);
+    ONEQUAD(5, 1, 13, 9);
+    ONEQUAD(10, 6, 2, 14);
+    ONEQUAD(15, 11, 7, 3);
+
+#undef ONEQUAD
+#undef ONEQUAD_SHUFFLE
+
+    for (i = 0; i < bytes; i++) {
+        c[i] = m[i] ^ partialblock[i];
+    }
+
+    sodium_memzero(partialblock, sizeof partialblock);
+}

data/vendor/libsodium/src/libsodium/crypto_stream/salsa20/xmm6int/u1.h

@@ -0,0 +1,207 @@
+while (bytes >= 64) {
+    __m128i diag0 = _mm_loadu_si128((__m128i *) (x + 0));
+    __m128i diag1 = _mm_loadu_si128((__m128i *) (x + 4));
+    __m128i diag2 = _mm_loadu_si128((__m128i *) (x + 8));
+    __m128i diag3 = _mm_loadu_si128((__m128i *) (x + 12));
+    __m128i a0, a1, a2, a3, a4, a5, a6, a7;
+    __m128i b0, b1, b2, b3, b4, b5, b6, b7;
+
+    uint32_t in8;
+    uint32_t in9;
+    int      i;
+
+    a0 = diag1;
+    for (i = 0; i < ROUNDS; i += 4) {
+        a0    = _mm_add_epi32(a0, diag0);
+        a1    = diag0;
+        b0    = a0;
+        a0    = _mm_slli_epi32(a0, 7);
+        b0    = _mm_srli_epi32(b0, 25);
+        diag3 = _mm_xor_si128(diag3, a0);
+
+        diag3 = _mm_xor_si128(diag3, b0);
+
+        a1    = _mm_add_epi32(a1, diag3);
+        a2    = diag3;
+        b1    = a1;
+        a1    = _mm_slli_epi32(a1, 9);
+        b1    = _mm_srli_epi32(b1, 23);
+        diag2 = _mm_xor_si128(diag2, a1);
+        diag3 = _mm_shuffle_epi32(diag3, 0x93);
+        diag2 = _mm_xor_si128(diag2, b1);
+
+        a2    = _mm_add_epi32(a2, diag2);
+        a3    = diag2;
+        b2    = a2;
+        a2    = _mm_slli_epi32(a2, 13);
+        b2    = _mm_srli_epi32(b2, 19);
+        diag1 = _mm_xor_si128(diag1, a2);
+        diag2 = _mm_shuffle_epi32(diag2, 0x4e);
+        diag1 = _mm_xor_si128(diag1, b2);
+
+        a3    = _mm_add_epi32(a3, diag1);
+        a4    = diag3;
+        b3    = a3;
+        a3    = _mm_slli_epi32(a3, 18);
+        b3    = _mm_srli_epi32(b3, 14);
+        diag0 = _mm_xor_si128(diag0, a3);
+        diag1 = _mm_shuffle_epi32(diag1, 0x39);
+        diag0 = _mm_xor_si128(diag0, b3);
+
+        a4    = _mm_add_epi32(a4, diag0);
+        a5    = diag0;
+        b4    = a4;
+        a4    = _mm_slli_epi32(a4, 7);
+        b4    = _mm_srli_epi32(b4, 25);
+        diag1 = _mm_xor_si128(diag1, a4);
+
+        diag1 = _mm_xor_si128(diag1, b4);
+
+        a5    = _mm_add_epi32(a5, diag1);
+        a6    = diag1;
+        b5    = a5;
+        a5    = _mm_slli_epi32(a5, 9);
+        b5    = _mm_srli_epi32(b5, 23);
+        diag2 = _mm_xor_si128(diag2, a5);
+        diag1 = _mm_shuffle_epi32(diag1, 0x93);
+        diag2 = _mm_xor_si128(diag2, b5);
+
+        a6    = _mm_add_epi32(a6, diag2);
+        a7    = diag2;
+        b6    = a6;
+        a6    = _mm_slli_epi32(a6, 13);
+        b6    = _mm_srli_epi32(b6, 19);
+        diag3 = _mm_xor_si128(diag3, a6);
+        diag2 = _mm_shuffle_epi32(diag2, 0x4e);
+        diag3 = _mm_xor_si128(diag3, b6);
+
+        a7    = _mm_add_epi32(a7, diag3);
+        a0    = diag1;
+        b7    = a7;
+        a7    = _mm_slli_epi32(a7, 18);
+        b7    = _mm_srli_epi32(b7, 14);
+        diag0 = _mm_xor_si128(diag0, a7);
+        diag3 = _mm_shuffle_epi32(diag3, 0x39);
+        diag0 = _mm_xor_si128(diag0, b7);
+
+        a0    = _mm_add_epi32(a0, diag0);
+        a1    = diag0;
+        b0    = a0;
+        a0    = _mm_slli_epi32(a0, 7);
+        b0    = _mm_srli_epi32(b0, 25);
+        diag3 = _mm_xor_si128(diag3, a0);
+
+        diag3 = _mm_xor_si128(diag3, b0);
+
+        a1    = _mm_add_epi32(a1, diag3);
+        a2    = diag3;
+        b1    = a1;
+        a1    = _mm_slli_epi32(a1, 9);
+        b1    = _mm_srli_epi32(b1, 23);
+        diag2 = _mm_xor_si128(diag2, a1);
+        diag3 = _mm_shuffle_epi32(diag3, 0x93);
+        diag2 = _mm_xor_si128(diag2, b1);
+
+        a2    = _mm_add_epi32(a2, diag2);
+        a3    = diag2;
+        b2    = a2;
+        a2    = _mm_slli_epi32(a2, 13);
+        b2    = _mm_srli_epi32(b2, 19);
+        diag1 = _mm_xor_si128(diag1, a2);
+        diag2 = _mm_shuffle_epi32(diag2, 0x4e);
+        diag1 = _mm_xor_si128(diag1, b2);
+
+        a3    = _mm_add_epi32(a3, diag1);
+        a4    = diag3;
+        b3    = a3;
+        a3    = _mm_slli_epi32(a3, 18);
+        b3    = _mm_srli_epi32(b3, 14);
+        diag0 = _mm_xor_si128(diag0, a3);
+        diag1 = _mm_shuffle_epi32(diag1, 0x39);
+        diag0 = _mm_xor_si128(diag0, b3);
+
+        a4    = _mm_add_epi32(a4, diag0);
+        a5    = diag0;
+        b4    = a4;
+        a4    = _mm_slli_epi32(a4, 7);
+        b4    = _mm_srli_epi32(b4, 25);
+        diag1 = _mm_xor_si128(diag1, a4);
+
+        diag1 = _mm_xor_si128(diag1, b4);
+
+        a5    = _mm_add_epi32(a5, diag1);
+        a6    = diag1;
+        b5    = a5;
+        a5    = _mm_slli_epi32(a5, 9);
+        b5    = _mm_srli_epi32(b5, 23);
+        diag2 = _mm_xor_si128(diag2, a5);
+        diag1 = _mm_shuffle_epi32(diag1, 0x93);
+        diag2 = _mm_xor_si128(diag2, b5);
+
+        a6    = _mm_add_epi32(a6, diag2);
+        a7    = diag2;
+        b6    = a6;
+        a6    = _mm_slli_epi32(a6, 13);
+        b6    = _mm_srli_epi32(b6, 19);
+        diag3 = _mm_xor_si128(diag3, a6);
+        diag2 = _mm_shuffle_epi32(diag2, 0x4e);
+        diag3 = _mm_xor_si128(diag3, b6);
+
+        a7    = _mm_add_epi32(a7, diag3);
+        a0    = diag1;
+        b7    = a7;
+        a7    = _mm_slli_epi32(a7, 18);
+        b7    = _mm_srli_epi32(b7, 14);
+        diag0 = _mm_xor_si128(diag0, a7);
+        diag3 = _mm_shuffle_epi32(diag3, 0x39);
+        diag0 = _mm_xor_si128(diag0, b7);
+    }
+
+    diag0 = _mm_add_epi32(diag0, _mm_loadu_si128((__m128i *) (x + 0)));
+    diag1 = _mm_add_epi32(diag1, _mm_loadu_si128((__m128i *) (x + 4)));
+    diag2 = _mm_add_epi32(diag2, _mm_loadu_si128((__m128i *) (x + 8)));
+    diag3 = _mm_add_epi32(diag3, _mm_loadu_si128((__m128i *) (x + 12)));
+
+#define ONEQUAD_SHUFFLE(A, B, C, D)                      \
+    do {                                                 \
+        uint32_t in##A = _mm_cvtsi128_si32(diag0);       \
+        uint32_t in##B = _mm_cvtsi128_si32(diag1);       \
+        uint32_t in##C = _mm_cvtsi128_si32(diag2);       \
+        uint32_t in##D = _mm_cvtsi128_si32(diag3);       \
+        diag0          = _mm_shuffle_epi32(diag0, 0x39); \
+        diag1          = _mm_shuffle_epi32(diag1, 0x39); \
+        diag2          = _mm_shuffle_epi32(diag2, 0x39); \
+        diag3          = _mm_shuffle_epi32(diag3, 0x39); \
+        in##A ^= *(uint32_t *) (m + (A * 4));            \
+        in##B ^= *(uint32_t *) (m + (B * 4));            \
+        in##C ^= *(uint32_t *) (m + (C * 4));            \
+        in##D ^= *(uint32_t *) (m + (D * 4));            \
+        *(uint32_t *) (c + (A * 4)) = in##A;             \
+        *(uint32_t *) (c + (B * 4)) = in##B;             \
+        *(uint32_t *) (c + (C * 4)) = in##C;             \
+        *(uint32_t *) (c + (D * 4)) = in##D;             \
+    } while (0)
+
+#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D)
+
+    ONEQUAD(0, 12, 8, 4);
+    ONEQUAD(5, 1, 13, 9);
+    ONEQUAD(10, 6, 2, 14);
+    ONEQUAD(15, 11, 7, 3);
+
+#undef ONEQUAD
+#undef ONEQUAD_SHUFFLE
+
+    in8 = x[8];
+    in9 = x[13];
+    in8++;
+    if (in8 == 0) {
+        in9++;
+    }
+    x[8]  = in8;
+    x[13] = in9;
+
+    c += 64;
+    m += 64;
+    bytes -= 64;
+}

data/vendor/libsodium/src/libsodium/crypto_stream/salsa20/xmm6int/u4.h

@@ -0,0 +1,547 @@
+if (bytes >= 256) {
+    __m128i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14,
+        y15;
+    __m128i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14,
+        z15;
+    __m128i orig0, orig1, orig2, orig3, orig4, orig5, orig6, orig7, orig8,
+        orig9, orig10, orig11, orig12, orig13, orig14, orig15;
+
+    uint32_t in8;
+    uint32_t in9;
+    int      i;
+
+    /* element broadcast immediate for _mm_shuffle_epi32 are in order:
+       0x00, 0x55, 0xaa, 0xff */
+    z0  = _mm_loadu_si128((__m128i *) (x + 0));
+    z5  = _mm_shuffle_epi32(z0, 0x55);
+    z10 = _mm_shuffle_epi32(z0, 0xaa);
+    z15 = _mm_shuffle_epi32(z0, 0xff);
+    z0  = _mm_shuffle_epi32(z0, 0x00);
+    z1  = _mm_loadu_si128((__m128i *) (x + 4));
+    z6  = _mm_shuffle_epi32(z1, 0xaa);
+    z11 = _mm_shuffle_epi32(z1, 0xff);
+    z12 = _mm_shuffle_epi32(z1, 0x00);
+    z1  = _mm_shuffle_epi32(z1, 0x55);
+    z2  = _mm_loadu_si128((__m128i *) (x + 8));
+    z7  = _mm_shuffle_epi32(z2, 0xff);
+    z13 = _mm_shuffle_epi32(z2, 0x55);
+    z2  = _mm_shuffle_epi32(z2, 0xaa);
+    /* no z8 -> first half of the nonce, will fill later */
+    z3  = _mm_loadu_si128((__m128i *) (x + 12));
+    z4  = _mm_shuffle_epi32(z3, 0x00);
+    z14 = _mm_shuffle_epi32(z3, 0xaa);
+    z3  = _mm_shuffle_epi32(z3, 0xff);
+    /* no z9 -> second half of the nonce, will fill later */
+    orig0  = z0;
+    orig1  = z1;
+    orig2  = z2;
+    orig3  = z3;
+    orig4  = z4;
+    orig5  = z5;
+    orig6  = z6;
+    orig7  = z7;
+    orig10 = z10;
+    orig11 = z11;
+    orig12 = z12;
+    orig13 = z13;
+    orig14 = z14;
+    orig15 = z15;
+
+    while (bytes >= 256) {
+        /* vector implementation for z8 and z9 */
+        /* not sure if it helps for only 4 blocks */
+        const __m128i addv8 = _mm_set_epi64x(1, 0);
+        const __m128i addv9 = _mm_set_epi64x(3, 2);
+        __m128i       t8, t9;
+        uint64_t      in89;
+
+        in8  = x[8];
+        in9  = x[13];
+        in89 = ((uint64_t) in8) | (((uint64_t) in9) << 32);
+        t8   = _mm_set1_epi64x(in89);
+        t9   = _mm_set1_epi64x(in89);
+
+        z8 = _mm_add_epi64(addv8, t8);
+        z9 = _mm_add_epi64(addv9, t9);
+
+        t8 = _mm_unpacklo_epi32(z8, z9);
+        t9 = _mm_unpackhi_epi32(z8, z9);
+
+        z8 = _mm_unpacklo_epi32(t8, t9);
+        z9 = _mm_unpackhi_epi32(t8, t9);
+
+        orig8 = z8;
+        orig9 = z9;
+
+        in89 += 4;
+
+        x[8]  = in89 & 0xFFFFFFFF;
+        x[13] = (in89 >> 32) & 0xFFFFFFFF;
+
+        z5  = orig5;
+        z10 = orig10;
+        z15 = orig15;
+        z14 = orig14;
+        z3  = orig3;
+        z6  = orig6;
+        z11 = orig11;
+        z1  = orig1;
+
+        z7  = orig7;
+        z13 = orig13;
+        z2  = orig2;
+        z9  = orig9;
+        z0  = orig0;
+        z12 = orig12;
+        z4  = orig4;
+        z8  = orig8;
+
+        for (i = 0; i < ROUNDS; i += 2) {
+            /* the inner loop is a direct translation (regexp search/replace)
+             * from the amd64-xmm6 ASM */
+            __m128i r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13,
+                r14, r15;
+
+            y4 = z12;
+            y4 = _mm_add_epi32(y4, z0);
+            r4 = y4;
+            y4 = _mm_slli_epi32(y4, 7);
+            z4 = _mm_xor_si128(z4, y4);
+            r4 = _mm_srli_epi32(r4, 25);
+            z4 = _mm_xor_si128(z4, r4);
+
+            y9 = z1;
+            y9 = _mm_add_epi32(y9, z5);
+            r9 = y9;
+            y9 = _mm_slli_epi32(y9, 7);
+            z9 = _mm_xor_si128(z9, y9);
+            r9 = _mm_srli_epi32(r9, 25);
+            z9 = _mm_xor_si128(z9, r9);
+
+            y8 = z0;
+            y8 = _mm_add_epi32(y8, z4);
+            r8 = y8;
+            y8 = _mm_slli_epi32(y8, 9);
+            z8 = _mm_xor_si128(z8, y8);
+            r8 = _mm_srli_epi32(r8, 23);
+            z8 = _mm_xor_si128(z8, r8);
+
+            y13 = z5;
+            y13 = _mm_add_epi32(y13, z9);
+            r13 = y13;
+            y13 = _mm_slli_epi32(y13, 9);
+            z13 = _mm_xor_si128(z13, y13);
+            r13 = _mm_srli_epi32(r13, 23);
+            z13 = _mm_xor_si128(z13, r13);
+
+            y12 = z4;
+            y12 = _mm_add_epi32(y12, z8);
+            r12 = y12;
+            y12 = _mm_slli_epi32(y12, 13);
+            z12 = _mm_xor_si128(z12, y12);
+            r12 = _mm_srli_epi32(r12, 19);
+            z12 = _mm_xor_si128(z12, r12);
+
+            y1 = z9;
+            y1 = _mm_add_epi32(y1, z13);
+            r1 = y1;
+            y1 = _mm_slli_epi32(y1, 13);
+            z1 = _mm_xor_si128(z1, y1);
+            r1 = _mm_srli_epi32(r1, 19);
+            z1 = _mm_xor_si128(z1, r1);
+
+            y0 = z8;
+            y0 = _mm_add_epi32(y0, z12);
+            r0 = y0;
+            y0 = _mm_slli_epi32(y0, 18);
+            z0 = _mm_xor_si128(z0, y0);
+            r0 = _mm_srli_epi32(r0, 14);
+            z0 = _mm_xor_si128(z0, r0);
+
+            y5 = z13;
+            y5 = _mm_add_epi32(y5, z1);
+            r5 = y5;
+            y5 = _mm_slli_epi32(y5, 18);
+            z5 = _mm_xor_si128(z5, y5);
+            r5 = _mm_srli_epi32(r5, 14);
+            z5 = _mm_xor_si128(z5, r5);
+
+            y14 = z6;
+            y14 = _mm_add_epi32(y14, z10);
+            r14 = y14;
+            y14 = _mm_slli_epi32(y14, 7);
+            z14 = _mm_xor_si128(z14, y14);
+            r14 = _mm_srli_epi32(r14, 25);
+            z14 = _mm_xor_si128(z14, r14);
+
+            y3 = z11;
+            y3 = _mm_add_epi32(y3, z15);
+            r3 = y3;
+            y3 = _mm_slli_epi32(y3, 7);
+            z3 = _mm_xor_si128(z3, y3);
+            r3 = _mm_srli_epi32(r3, 25);
+            z3 = _mm_xor_si128(z3, r3);
+
+            y2 = z10;
+            y2 = _mm_add_epi32(y2, z14);
+            r2 = y2;
+            y2 = _mm_slli_epi32(y2, 9);
+            z2 = _mm_xor_si128(z2, y2);
+            r2 = _mm_srli_epi32(r2, 23);
+            z2 = _mm_xor_si128(z2, r2);
+
+            y7 = z15;
+            y7 = _mm_add_epi32(y7, z3);
+            r7 = y7;
+            y7 = _mm_slli_epi32(y7, 9);
+            z7 = _mm_xor_si128(z7, y7);
+            r7 = _mm_srli_epi32(r7, 23);
+            z7 = _mm_xor_si128(z7, r7);
+
+            y6 = z14;
+            y6 = _mm_add_epi32(y6, z2);
+            r6 = y6;
+            y6 = _mm_slli_epi32(y6, 13);
+            z6 = _mm_xor_si128(z6, y6);
+            r6 = _mm_srli_epi32(r6, 19);
+            z6 = _mm_xor_si128(z6, r6);
+
+            y11 = z3;
+            y11 = _mm_add_epi32(y11, z7);
+            r11 = y11;
+            y11 = _mm_slli_epi32(y11, 13);
+            z11 = _mm_xor_si128(z11, y11);
+            r11 = _mm_srli_epi32(r11, 19);
+            z11 = _mm_xor_si128(z11, r11);
+
+            y10 = z2;
+            y10 = _mm_add_epi32(y10, z6);
+            r10 = y10;
+            y10 = _mm_slli_epi32(y10, 18);
+            z10 = _mm_xor_si128(z10, y10);
+            r10 = _mm_srli_epi32(r10, 14);
+            z10 = _mm_xor_si128(z10, r10);
+
+            y1 = z3;
+            y1 = _mm_add_epi32(y1, z0);
+            r1 = y1;
+            y1 = _mm_slli_epi32(y1, 7);
+            z1 = _mm_xor_si128(z1, y1);
+            r1 = _mm_srli_epi32(r1, 25);
+            z1 = _mm_xor_si128(z1, r1);
+
+            y15 = z7;
+            y15 = _mm_add_epi32(y15, z11);
+            r15 = y15;
+            y15 = _mm_slli_epi32(y15, 18);
+            z15 = _mm_xor_si128(z15, y15);
+            r15 = _mm_srli_epi32(r15, 14);
+            z15 = _mm_xor_si128(z15, r15);
+
+            y6 = z4;
+            y6 = _mm_add_epi32(y6, z5);
+            r6 = y6;
+            y6 = _mm_slli_epi32(y6, 7);
+            z6 = _mm_xor_si128(z6, y6);
+            r6 = _mm_srli_epi32(r6, 25);
+            z6 = _mm_xor_si128(z6, r6);
+
+            y2 = z0;
+            y2 = _mm_add_epi32(y2, z1);
+            r2 = y2;
+            y2 = _mm_slli_epi32(y2, 9);
+            z2 = _mm_xor_si128(z2, y2);
+            r2 = _mm_srli_epi32(r2, 23);
+            z2 = _mm_xor_si128(z2, r2);
+
+            y7 = z5;
+            y7 = _mm_add_epi32(y7, z6);
+            r7 = y7;
+            y7 = _mm_slli_epi32(y7, 9);
+            z7 = _mm_xor_si128(z7, y7);
+            r7 = _mm_srli_epi32(r7, 23);
+            z7 = _mm_xor_si128(z7, r7);
+
+            y3 = z1;
+            y3 = _mm_add_epi32(y3, z2);
+            r3 = y3;
+            y3 = _mm_slli_epi32(y3, 13);
+            z3 = _mm_xor_si128(z3, y3);
+            r3 = _mm_srli_epi32(r3, 19);
+            z3 = _mm_xor_si128(z3, r3);
+
+            y4 = z6;
+            y4 = _mm_add_epi32(y4, z7);
+            r4 = y4;
+            y4 = _mm_slli_epi32(y4, 13);
+            z4 = _mm_xor_si128(z4, y4);
+            r4 = _mm_srli_epi32(r4, 19);
+            z4 = _mm_xor_si128(z4, r4);
+
+            y0 = z2;
+            y0 = _mm_add_epi32(y0, z3);
+            r0 = y0;
+            y0 = _mm_slli_epi32(y0, 18);
+            z0 = _mm_xor_si128(z0, y0);
+            r0 = _mm_srli_epi32(r0, 14);
+            z0 = _mm_xor_si128(z0, r0);
+
+            y5 = z7;
+            y5 = _mm_add_epi32(y5, z4);
+            r5 = y5;
+            y5 = _mm_slli_epi32(y5, 18);
+            z5 = _mm_xor_si128(z5, y5);
+            r5 = _mm_srli_epi32(r5, 14);
+            z5 = _mm_xor_si128(z5, r5);
+
+            y11 = z9;
+            y11 = _mm_add_epi32(y11, z10);
+            r11 = y11;
+            y11 = _mm_slli_epi32(y11, 7);
+            z11 = _mm_xor_si128(z11, y11);
+            r11 = _mm_srli_epi32(r11, 25);
+            z11 = _mm_xor_si128(z11, r11);
+
+            y12 = z14;
+            y12 = _mm_add_epi32(y12, z15);
+            r12 = y12;
+            y12 = _mm_slli_epi32(y12, 7);
+            z12 = _mm_xor_si128(z12, y12);
+            r12 = _mm_srli_epi32(r12, 25);
+            z12 = _mm_xor_si128(z12, r12);
+
+            y8 = z10;
+            y8 = _mm_add_epi32(y8, z11);
+            r8 = y8;
+            y8 = _mm_slli_epi32(y8, 9);
+            z8 = _mm_xor_si128(z8, y8);
+            r8 = _mm_srli_epi32(r8, 23);
+            z8 = _mm_xor_si128(z8, r8);
+
+            y13 = z15;
+            y13 = _mm_add_epi32(y13, z12);
+            r13 = y13;
+            y13 = _mm_slli_epi32(y13, 9);
+            z13 = _mm_xor_si128(z13, y13);
+            r13 = _mm_srli_epi32(r13, 23);
+            z13 = _mm_xor_si128(z13, r13);
+
+            y9 = z11;
+            y9 = _mm_add_epi32(y9, z8);
+            r9 = y9;
+            y9 = _mm_slli_epi32(y9, 13);
+            z9 = _mm_xor_si128(z9, y9);
+            r9 = _mm_srli_epi32(r9, 19);
+            z9 = _mm_xor_si128(z9, r9);
+
+            y14 = z12;
+            y14 = _mm_add_epi32(y14, z13);
+            r14 = y14;
+            y14 = _mm_slli_epi32(y14, 13);
+            z14 = _mm_xor_si128(z14, y14);
+            r14 = _mm_srli_epi32(r14, 19);
+            z14 = _mm_xor_si128(z14, r14);
+
+            y10 = z8;
+            y10 = _mm_add_epi32(y10, z9);
+            r10 = y10;
+            y10 = _mm_slli_epi32(y10, 18);
+            z10 = _mm_xor_si128(z10, y10);
+            r10 = _mm_srli_epi32(r10, 14);
+            z10 = _mm_xor_si128(z10, r10);
+
+            y15 = z13;
+            y15 = _mm_add_epi32(y15, z14);
+            r15 = y15;
+            y15 = _mm_slli_epi32(y15, 18);
+            z15 = _mm_xor_si128(z15, y15);
+            r15 = _mm_srli_epi32(r15, 14);
+            z15 = _mm_xor_si128(z15, r15);
+        }
+
+/* store data ; this macro replicates the original amd64-xmm6 code */
+#define ONEQUAD_SHUFFLE(A, B, C, D)        \
+    z##A  = _mm_add_epi32(z##A, orig##A);  \
+    z##B  = _mm_add_epi32(z##B, orig##B);  \
+    z##C  = _mm_add_epi32(z##C, orig##C);  \
+    z##D  = _mm_add_epi32(z##D, orig##D);  \
+    in##A = _mm_cvtsi128_si32(z##A);       \
+    in##B = _mm_cvtsi128_si32(z##B);       \
+    in##C = _mm_cvtsi128_si32(z##C);       \
+    in##D = _mm_cvtsi128_si32(z##D);       \
+    z##A  = _mm_shuffle_epi32(z##A, 0x39); \
+    z##B  = _mm_shuffle_epi32(z##B, 0x39); \
+    z##C  = _mm_shuffle_epi32(z##C, 0x39); \
+    z##D  = _mm_shuffle_epi32(z##D, 0x39); \
+                                           \
+    in##A ^= *(uint32_t *) (m + 0);        \
+    in##B ^= *(uint32_t *) (m + 4);        \
+    in##C ^= *(uint32_t *) (m + 8);        \
+    in##D ^= *(uint32_t *) (m + 12);       \
+                                           \
+    *(uint32_t *) (c + 0)  = in##A;        \
+    *(uint32_t *) (c + 4)  = in##B;        \
+    *(uint32_t *) (c + 8)  = in##C;        \
+    *(uint32_t *) (c + 12) = in##D;        \
+                                           \
+    in##A = _mm_cvtsi128_si32(z##A);       \
+    in##B = _mm_cvtsi128_si32(z##B);       \
+    in##C = _mm_cvtsi128_si32(z##C);       \
+    in##D = _mm_cvtsi128_si32(z##D);       \
+    z##A  = _mm_shuffle_epi32(z##A, 0x39); \
+    z##B  = _mm_shuffle_epi32(z##B, 0x39); \
+    z##C  = _mm_shuffle_epi32(z##C, 0x39); \
+    z##D  = _mm_shuffle_epi32(z##D, 0x39); \
+                                           \
+    in##A ^= *(uint32_t *) (m + 64);       \
+    in##B ^= *(uint32_t *) (m + 68);       \
+    in##C ^= *(uint32_t *) (m + 72);       \
+    in##D ^= *(uint32_t *) (m + 76);       \
+    *(uint32_t *) (c + 64) = in##A;        \
+    *(uint32_t *) (c + 68) = in##B;        \
+    *(uint32_t *) (c + 72) = in##C;        \
+    *(uint32_t *) (c + 76) = in##D;        \
+                                           \
+    in##A = _mm_cvtsi128_si32(z##A);       \
+    in##B = _mm_cvtsi128_si32(z##B);       \
+    in##C = _mm_cvtsi128_si32(z##C);       \
+    in##D = _mm_cvtsi128_si32(z##D);       \
+    z##A  = _mm_shuffle_epi32(z##A, 0x39); \
+    z##B  = _mm_shuffle_epi32(z##B, 0x39); \
+    z##C  = _mm_shuffle_epi32(z##C, 0x39); \
+    z##D  = _mm_shuffle_epi32(z##D, 0x39); \
+                                           \
+    in##A ^= *(uint32_t *) (m + 128);      \
+    in##B ^= *(uint32_t *) (m + 132);      \
+    in##C ^= *(uint32_t *) (m + 136);      \
+    in##D ^= *(uint32_t *) (m + 140);      \
+    *(uint32_t *) (c + 128) = in##A;       \
+    *(uint32_t *) (c + 132) = in##B;       \
+    *(uint32_t *) (c + 136) = in##C;       \
+    *(uint32_t *) (c + 140) = in##D;       \
+                                           \
+    in##A = _mm_cvtsi128_si32(z##A);       \
+    in##B = _mm_cvtsi128_si32(z##B);       \
+    in##C = _mm_cvtsi128_si32(z##C);       \
+    in##D = _mm_cvtsi128_si32(z##D);       \
+                                           \
+    in##A ^= *(uint32_t *) (m + 192);      \
+    in##B ^= *(uint32_t *) (m + 196);      \
+    in##C ^= *(uint32_t *) (m + 200);      \
+    in##D ^= *(uint32_t *) (m + 204);      \
+    *(uint32_t *) (c + 192) = in##A;       \
+    *(uint32_t *) (c + 196) = in##B;       \
+    *(uint32_t *) (c + 200) = in##C;       \
+    *(uint32_t *) (c + 204) = in##D
+
+/* store data ; this macro replaces shuffle+mov by a direct extract; not much
+ * difference */
+#define ONEQUAD_EXTRACT(A, B, C, D)       \
+    z##A  = _mm_add_epi32(z##A, orig##A); \
+    z##B  = _mm_add_epi32(z##B, orig##B); \
+    z##C  = _mm_add_epi32(z##C, orig##C); \
+    z##D  = _mm_add_epi32(z##D, orig##D); \
+    in##A = _mm_cvtsi128_si32(z##A);      \
+    in##B = _mm_cvtsi128_si32(z##B);      \
+    in##C = _mm_cvtsi128_si32(z##C);      \
+    in##D = _mm_cvtsi128_si32(z##D);      \
+    in##A ^= *(uint32_t *) (m + 0);       \
+    in##B ^= *(uint32_t *) (m + 4);       \
+    in##C ^= *(uint32_t *) (m + 8);       \
+    in##D ^= *(uint32_t *) (m + 12);      \
+    *(uint32_t *) (c + 0)  = in##A;       \
+    *(uint32_t *) (c + 4)  = in##B;       \
+    *(uint32_t *) (c + 8)  = in##C;       \
+    *(uint32_t *) (c + 12) = in##D;       \
+                                          \
+    in##A = _mm_extract_epi32(z##A, 1);   \
+    in##B = _mm_extract_epi32(z##B, 1);   \
+    in##C = _mm_extract_epi32(z##C, 1);   \
+    in##D = _mm_extract_epi32(z##D, 1);   \
+                                          \
+    in##A ^= *(uint32_t *) (m + 64);      \
+    in##B ^= *(uint32_t *) (m + 68);      \
+    in##C ^= *(uint32_t *) (m + 72);      \
+    in##D ^= *(uint32_t *) (m + 76);      \
+    *(uint32_t *) (c + 64) = in##A;       \
+    *(uint32_t *) (c + 68) = in##B;       \
+    *(uint32_t *) (c + 72) = in##C;       \
+    *(uint32_t *) (c + 76) = in##D;       \
+                                          \
+    in##A = _mm_extract_epi32(z##A, 2);   \
+    in##B = _mm_extract_epi32(z##B, 2);   \
+    in##C = _mm_extract_epi32(z##C, 2);   \
+    in##D = _mm_extract_epi32(z##D, 2);   \
+                                          \
+    in##A ^= *(uint32_t *) (m + 128);     \
+    in##B ^= *(uint32_t *) (m + 132);     \
+    in##C ^= *(uint32_t *) (m + 136);     \
+    in##D ^= *(uint32_t *) (m + 140);     \
+    *(uint32_t *) (c + 128) = in##A;      \
+    *(uint32_t *) (c + 132) = in##B;      \
+    *(uint32_t *) (c + 136) = in##C;      \
+    *(uint32_t *) (c + 140) = in##D;      \
+                                          \
+    in##A = _mm_extract_epi32(z##A, 3);   \
+    in##B = _mm_extract_epi32(z##B, 3);   \
+    in##C = _mm_extract_epi32(z##C, 3);   \
+    in##D = _mm_extract_epi32(z##D, 3);   \
+                                          \
+    in##A ^= *(uint32_t *) (m + 192);     \
+    in##B ^= *(uint32_t *) (m + 196);     \
+    in##C ^= *(uint32_t *) (m + 200);     \
+    in##D ^= *(uint32_t *) (m + 204);     \
+    *(uint32_t *) (c + 192) = in##A;      \
+    *(uint32_t *) (c + 196) = in##B;      \
+    *(uint32_t *) (c + 200) = in##C;      \
+    *(uint32_t *) (c + 204) = in##D
+
+/* store data ; this macro first transpose data in-registers, and then store
+ * them in memory. much faster with icc. */
+#define ONEQUAD_TRANSPOSE(A, B, C, D)                                   \
+    z##A = _mm_add_epi32(z##A, orig##A);                                \
+    z##B = _mm_add_epi32(z##B, orig##B);                                \
+    z##C = _mm_add_epi32(z##C, orig##C);                                \
+    z##D = _mm_add_epi32(z##D, orig##D);                                \
+    y##A = _mm_unpacklo_epi32(z##A, z##B);                              \
+    y##B = _mm_unpacklo_epi32(z##C, z##D);                              \
+    y##C = _mm_unpackhi_epi32(z##A, z##B);                              \
+    y##D = _mm_unpackhi_epi32(z##C, z##D);                              \
+    z##A = _mm_unpacklo_epi64(y##A, y##B);                              \
+    z##B = _mm_unpackhi_epi64(y##A, y##B);                              \
+    z##C = _mm_unpacklo_epi64(y##C, y##D);                              \
+    z##D = _mm_unpackhi_epi64(y##C, y##D);                              \
+    y##A = _mm_xor_si128(z##A, _mm_loadu_si128((__m128i *) (m + 0)));   \
+    _mm_storeu_si128((__m128i *) (c + 0), y##A);                        \
+    y##B = _mm_xor_si128(z##B, _mm_loadu_si128((__m128i *) (m + 64)));  \
+    _mm_storeu_si128((__m128i *) (c + 64), y##B);                       \
+    y##C = _mm_xor_si128(z##C, _mm_loadu_si128((__m128i *) (m + 128))); \
+    _mm_storeu_si128((__m128i *) (c + 128), y##C);                      \
+    y##D = _mm_xor_si128(z##D, _mm_loadu_si128((__m128i *) (m + 192))); \
+    _mm_storeu_si128((__m128i *) (c + 192), y##D)
+
+#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D)
+
+        ONEQUAD(0, 1, 2, 3);
+        m += 16;
+        c += 16;
+        ONEQUAD(4, 5, 6, 7);
+        m += 16;
+        c += 16;
+        ONEQUAD(8, 9, 10, 11);
+        m += 16;
+        c += 16;
+        ONEQUAD(12, 13, 14, 15);
+        m -= 48;
+        c -= 48;
+
+#undef ONEQUAD
+#undef ONEQUAD_TRANSPOSE
+#undef ONEQUAD_EXTRACT
+#undef ONEQUAD_SHUFFLE
+
+        bytes -= 256;
+        c += 256;
+        m += 256;
+    }
+}