rbnacl-libsodium 1.0.6 → 1.0.7

data/vendor/libsodium/src/libsodium/crypto_stream/aes128ctr/portable/afternm_aes128ctr.c

@@ -2,12 +2,12 @@
  * Date: 2009-03-19
  * Public domain */
 
-#include "api.h"
+#include "crypto_stream_aes128ctr.h"
 #include "int128.h"
 #include "common.h"
 #include "consts.h"
 
-int crypto_stream_afternm(unsigned char *out, unsigned long long len, const unsigned char *nonce, const unsigned char *c)
+int crypto_stream_aes128ctr_afternm(unsigned char *out, unsigned long long len, const unsigned char *nonce, const unsigned char *c)
 {
 
   int128 xmm0;

data/vendor/libsodium/src/libsodium/crypto_stream/aes128ctr/portable/beforenm_aes128ctr.c

@@ -2,12 +2,12 @@
  * Date: 2009-03-19
  * Public domain */
 
-#include "api.h"
+#include "crypto_stream_aes128ctr.h"
 #include "consts.h"
 #include "int128.h"
 #include "common.h"
 
-int crypto_stream_beforenm(unsigned char *c, const unsigned char *k)
+int crypto_stream_aes128ctr_beforenm(unsigned char *c, const unsigned char *k)
 {
 
   /*

data/vendor/libsodium/src/libsodium/crypto_stream/aes128ctr/portable/stream_aes128ctr.c

@@ -1,19 +1,20 @@
-#include "api.h"
 
-int crypto_stream(
+#include "crypto_stream_aes128ctr.h"
+
+int crypto_stream_aes128ctr(
         unsigned char *out,
         unsigned long long outlen,
         const unsigned char *n,
         const unsigned char *k
         )
 {
-    unsigned char d[crypto_stream_BEFORENMBYTES];
-    crypto_stream_beforenm(d, k);
-    crypto_stream_afternm(out, outlen, n, d);
+    unsigned char d[crypto_stream_aes128ctr_BEFORENMBYTES];
+    crypto_stream_aes128ctr_beforenm(d, k);
+    crypto_stream_aes128ctr_afternm(out, outlen, n, d);
     return 0;
 }
 
-int crypto_stream_xor(
+int crypto_stream_aes128ctr_xor(
         unsigned char *out,
         const unsigned char *in,
         unsigned long long inlen,
@@ -21,8 +22,8 @@ int crypto_stream_xor(
         const unsigned char *k
         )
 {
-    unsigned char d[crypto_stream_BEFORENMBYTES];
-    crypto_stream_beforenm(d, k);
-    crypto_stream_xor_afternm(out, in, inlen, n, d);
+    unsigned char d[crypto_stream_aes128ctr_BEFORENMBYTES];
+    crypto_stream_aes128ctr_beforenm(d, k);
+    crypto_stream_aes128ctr_xor_afternm(out, in, inlen, n, d);
     return 0;
 }

data/vendor/libsodium/src/libsodium/crypto_stream/aes128ctr/portable/xor_afternm_aes128ctr.c

@@ -2,13 +2,12 @@
  * Date: 2009-03-19
  * Public domain */
 
-#include <stdio.h>
-#include "api.h"
+#include "crypto_stream_aes128ctr.h"
 #include "int128.h"
 #include "common.h"
 #include "consts.h"
 
-int crypto_stream_xor_afternm(unsigned char *out, const unsigned char *in, unsigned long long len, const unsigned char *nonce, const unsigned char *c)
+int crypto_stream_aes128ctr_xor_afternm(unsigned char *out, const unsigned char *in, unsigned long long len, const unsigned char *nonce, const unsigned char *c)
 {
 
   int128 xmm0;

data/vendor/libsodium/src/libsodium/crypto_stream/chacha20/ref/stream_chacha20_ref.c

@@ -1,6 +1,4 @@
 
-/* $OpenBSD: chacha.c,v 1.1 2013/11/21 00:45:44 djm Exp $ */
-
 /*
  chacha-merged.c version 20080118
  D. J. Bernstein
@@ -11,9 +9,10 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "api.h"
-#include "crypto_stream_chacha20.h"
 #include "utils.h"
+#include "crypto_stream_chacha20.h"
+#include "stream_chacha20_ref.h"
+#include "../stream_chacha20.h"
 
 struct chacha_ctx {
     uint32_t input[16];
@@ -63,46 +62,46 @@ static const unsigned char sigma[16] = {
 };
 
 static void
-chacha_keysetup(chacha_ctx *x, const u8 *k)
+chacha_keysetup(chacha_ctx *ctx, const u8 *k)
 {
     const unsigned char *constants;
 
-    x->input[4] = U8TO32_LITTLE(k + 0);
-    x->input[5] = U8TO32_LITTLE(k + 4);
-    x->input[6] = U8TO32_LITTLE(k + 8);
-    x->input[7] = U8TO32_LITTLE(k + 12);
+    ctx->input[4] = U8TO32_LITTLE(k + 0);
+    ctx->input[5] = U8TO32_LITTLE(k + 4);
+    ctx->input[6] = U8TO32_LITTLE(k + 8);
+    ctx->input[7] = U8TO32_LITTLE(k + 12);
     k += 16;
     constants = sigma;
-    x->input[8] = U8TO32_LITTLE(k + 0);
-    x->input[9] = U8TO32_LITTLE(k + 4);
-    x->input[10] = U8TO32_LITTLE(k + 8);
-    x->input[11] = U8TO32_LITTLE(k + 12);
-    x->input[0] = U8TO32_LITTLE(constants + 0);
-    x->input[1] = U8TO32_LITTLE(constants + 4);
-    x->input[2] = U8TO32_LITTLE(constants + 8);
-    x->input[3] = U8TO32_LITTLE(constants + 12);
+    ctx->input[8] = U8TO32_LITTLE(k + 0);
+    ctx->input[9] = U8TO32_LITTLE(k + 4);
+    ctx->input[10] = U8TO32_LITTLE(k + 8);
+    ctx->input[11] = U8TO32_LITTLE(k + 12);
+    ctx->input[0] = U8TO32_LITTLE(constants + 0);
+    ctx->input[1] = U8TO32_LITTLE(constants + 4);
+    ctx->input[2] = U8TO32_LITTLE(constants + 8);
+    ctx->input[3] = U8TO32_LITTLE(constants + 12);
 }
 
 static void
-chacha_ivsetup(chacha_ctx *x, const u8 *iv, const u8 *counter)
+chacha_ivsetup(chacha_ctx *ctx, const u8 *iv, const u8 *counter)
 {
-    x->input[12] = counter == NULL ? 0 : U8TO32_LITTLE(counter + 0);
-    x->input[13] = counter == NULL ? 0 : U8TO32_LITTLE(counter + 4);
-    x->input[14] = U8TO32_LITTLE(iv + 0);
-    x->input[15] = U8TO32_LITTLE(iv + 4);
+    ctx->input[12] = counter == NULL ? 0 : U8TO32_LITTLE(counter + 0);
+    ctx->input[13] = counter == NULL ? 0 : U8TO32_LITTLE(counter + 4);
+    ctx->input[14] = U8TO32_LITTLE(iv + 0);
+    ctx->input[15] = U8TO32_LITTLE(iv + 4);
 }
 
 static void
-chacha_ietf_ivsetup(chacha_ctx *x, const u8 *iv, const u8 *counter)
+chacha_ietf_ivsetup(chacha_ctx *ctx, const u8 *iv, const u8 *counter)
 {
-    x->input[12] = counter == NULL ? 0 : U8TO32_LITTLE(counter);
-    x->input[13] = U8TO32_LITTLE(iv + 0);
-    x->input[14] = U8TO32_LITTLE(iv + 4);
-    x->input[15] = U8TO32_LITTLE(iv + 8);
+    ctx->input[12] = counter == NULL ? 0 : U8TO32_LITTLE(counter);
+    ctx->input[13] = U8TO32_LITTLE(iv + 0);
+    ctx->input[14] = U8TO32_LITTLE(iv + 4);
+    ctx->input[15] = U8TO32_LITTLE(iv + 8);
 }
 
 static void
-chacha_encrypt_bytes(chacha_ctx *x, const u8 *m, u8 *c, unsigned long long bytes)
+chacha_encrypt_bytes(chacha_ctx *ctx, const u8 *m, u8 *c, unsigned long long bytes)
 {
     u32 x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
     u32 j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15;
@@ -113,22 +112,25 @@ chacha_encrypt_bytes(chacha_ctx *x, const u8 *m, u8 *c, unsigned long long bytes
     if (!bytes) {
         return; /* LCOV_EXCL_LINE */
     }
-    j0 = x->input[0];
-    j1 = x->input[1];
-    j2 = x->input[2];
-    j3 = x->input[3];
-    j4 = x->input[4];
-    j5 = x->input[5];
-    j6 = x->input[6];
-    j7 = x->input[7];
-    j8 = x->input[8];
-    j9 = x->input[9];
-    j10 = x->input[10];
-    j11 = x->input[11];
-    j12 = x->input[12];
-    j13 = x->input[13];
-    j14 = x->input[14];
-    j15 = x->input[15];
+    if (bytes > 64ULL * (1ULL << 32) - 64ULL) {
+        abort();
+    }
+    j0 = ctx->input[0];
+    j1 = ctx->input[1];
+    j2 = ctx->input[2];
+    j3 = ctx->input[3];
+    j4 = ctx->input[4];
+    j5 = ctx->input[5];
+    j6 = ctx->input[6];
+    j7 = ctx->input[7];
+    j8 = ctx->input[8];
+    j9 = ctx->input[9];
+    j10 = ctx->input[10];
+    j11 = ctx->input[11];
+    j12 = ctx->input[12];
+    j13 = ctx->input[13];
+    j14 = ctx->input[14];
+    j15 = ctx->input[15];
 
     for (;;) {
         if (bytes < 64) {
@@ -230,8 +232,8 @@ chacha_encrypt_bytes(chacha_ctx *x, const u8 *m, u8 *c, unsigned long long bytes
                     ctarget[i] = c[i];
                 }
             }
-            x->input[12] = j12;
-            x->input[13] = j13;
+            ctx->input[12] = j12;
+            ctx->input[13] = j13;
             return;
         }
         bytes -= 64;
@@ -240,9 +242,9 @@ chacha_encrypt_bytes(chacha_ctx *x, const u8 *m, u8 *c, unsigned long long bytes
     }
 }
 
-int
-crypto_stream_chacha20_ref(unsigned char *c, unsigned long long clen,
-                           const unsigned char *n, const unsigned char *k)
+static int
+stream_ref(unsigned char *c, unsigned long long clen,
+           const unsigned char *n, const unsigned char *k)
 {
     struct chacha_ctx ctx;
 
@@ -259,18 +261,15 @@ crypto_stream_chacha20_ref(unsigned char *c, unsigned long long clen,
     return 0;
 }
 
-int
-crypto_stream_chacha20_ietf_ref(unsigned char *c, unsigned long long clen,
-                                const unsigned char *n, const unsigned char *k)
+static int
+stream_ietf_ref(unsigned char *c, unsigned long long clen,
+                const unsigned char *n, const unsigned char *k)
 {
     struct chacha_ctx ctx;
 
     if (!clen) {
         return 0;
     }
-    if (clen > 64ULL * (1ULL << 32) - 64ULL) {
-        abort();
-    }
     (void) sizeof(int[crypto_stream_chacha20_KEYBYTES == 256 / 8 ? 1 : -1]);
     chacha_keysetup(&ctx, k);
     chacha_ietf_ivsetup(&ctx, n, NULL);
@@ -281,11 +280,11 @@ crypto_stream_chacha20_ietf_ref(unsigned char *c, unsigned long long clen,
     return 0;
 }
 
-int
-crypto_stream_chacha20_ref_xor_ic(unsigned char *c, const unsigned char *m,
-                                  unsigned long long mlen,
-                                  const unsigned char *n, uint64_t ic,
-                                  const unsigned char *k)
+static int
+stream_ref_xor_ic(unsigned char *c, const unsigned char *m,
+                  unsigned long long mlen,
+                  const unsigned char *n, uint64_t ic,
+                  const unsigned char *k)
 {
     struct chacha_ctx ctx;
     uint8_t           ic_bytes[8];
@@ -302,18 +301,16 @@ crypto_stream_chacha20_ref_xor_ic(unsigned char *c, const unsigned char *m,
     chacha_keysetup(&ctx, k);
     chacha_ivsetup(&ctx, n, ic_bytes);
     chacha_encrypt_bytes(&ctx, m, c, mlen);
-
     sodium_memzero(&ctx, sizeof ctx);
-    sodium_memzero(ic_bytes, sizeof ic_bytes);
 
     return 0;
 }
 
-int
-crypto_stream_chacha20_ietf_ref_xor_ic(unsigned char *c, const unsigned char *m,
-                                       unsigned long long mlen,
-                                       const unsigned char *n, uint32_t ic,
-                                       const unsigned char *k)
+static int
+stream_ietf_ref_xor_ic(unsigned char *c, const unsigned char *m,
+                       unsigned long long mlen,
+                       const unsigned char *n, uint32_t ic,
+                       const unsigned char *k)
 {
     struct chacha_ctx ctx;
     uint8_t           ic_bytes[4];
@@ -326,7 +323,14 @@ crypto_stream_chacha20_ietf_ref_xor_ic(unsigned char *c, const unsigned char *m,
     chacha_ietf_ivsetup(&ctx, n, ic_bytes);
     chacha_encrypt_bytes(&ctx, m, c, mlen);
     sodium_memzero(&ctx, sizeof ctx);
-    sodium_memzero(ic_bytes, sizeof ic_bytes);
 
     return 0;
 }
+
+struct crypto_stream_chacha20_implementation
+crypto_stream_chacha20_ref_implementation = {
+    SODIUM_C99(.stream =) stream_ref,
+    SODIUM_C99(.stream_ietf =) stream_ietf_ref,
+    SODIUM_C99(.stream_xor_ic =) stream_ref_xor_ic,
+    SODIUM_C99(.stream_ietf_xor_ic =) stream_ietf_ref_xor_ic
+};

data/vendor/libsodium/src/libsodium/crypto_stream/chacha20/ref/{api.h → stream_chacha20_ref.h}

@@ -1,6 +1,11 @@
 
+#include <stdint.h>
+
 #include "crypto_stream_chacha20.h"
 
+extern struct crypto_stream_chacha20_implementation
+    crypto_stream_chacha20_ref_implementation;
+
 int
 crypto_stream_chacha20_ref(unsigned char *c, unsigned long long clen,
                            const unsigned char *n, const unsigned char *k);

data/vendor/libsodium/src/libsodium/crypto_stream/chacha20/{stream_chacha20_api.c → stream_chacha20.c}

@@ -1,5 +1,13 @@
 #include "crypto_stream_chacha20.h"
-#include "ref/api.h"
+#include "stream_chacha20.h"
+#include "runtime.h"
+#include "ref/stream_chacha20_ref.h"
+#if (defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) && defined(__GNUC__))
+# include "vec/stream_chacha20_vec.h"
+#endif
+
+static const crypto_stream_chacha20_implementation *implementation =
+    &crypto_stream_chacha20_ref_implementation;
 
 size_t
 crypto_stream_chacha20_keybytes(void) {
@@ -20,14 +28,14 @@ int
 crypto_stream_chacha20(unsigned char *c, unsigned long long clen,
                        const unsigned char *n, const unsigned char *k)
 {
-    return crypto_stream_chacha20_ref(c, clen, n, k);
+    return implementation->stream(c, clen, n, k);
 }
 
 int
 crypto_stream_chacha20_ietf(unsigned char *c, unsigned long long clen,
                             const unsigned char *n, const unsigned char *k)
 {
-    return crypto_stream_chacha20_ietf_ref(c, clen, n, k);
+    return implementation->stream_ietf(c, clen, n, k);
 }
 
 int
@@ -36,7 +44,7 @@ crypto_stream_chacha20_xor_ic(unsigned char *c, const unsigned char *m,
                               const unsigned char *n, uint64_t ic,
                               const unsigned char *k)
 {
-    return crypto_stream_chacha20_ref_xor_ic(c, m, mlen, n, ic, k);
+    return implementation->stream_xor_ic(c, m, mlen, n, ic, k);
 }
 
 int
@@ -45,7 +53,7 @@ crypto_stream_chacha20_ietf_xor_ic(unsigned char *c, const unsigned char *m,
                                    const unsigned char *n, uint32_t ic,
                                    const unsigned char *k)
 {
-    return crypto_stream_chacha20_ietf_ref_xor_ic(c, m, mlen, n, ic, k);
+    return implementation->stream_ietf_xor_ic(c, m, mlen, n, ic, k);
 }
 
 int
@@ -53,7 +61,7 @@ crypto_stream_chacha20_xor(unsigned char *c, const unsigned char *m,
                            unsigned long long mlen, const unsigned char *n,
                            const unsigned char *k)
 {
-    return crypto_stream_chacha20_ref_xor_ic(c, m, mlen, n, 0U, k);
+    return implementation->stream_xor_ic(c, m, mlen, n, 0U, k);
 }
 
 int
@@ -61,5 +69,17 @@ crypto_stream_chacha20_ietf_xor(unsigned char *c, const unsigned char *m,
                                 unsigned long long mlen, const unsigned char *n,
                                 const unsigned char *k)
 {
-    return crypto_stream_chacha20_ietf_ref_xor_ic(c, m, mlen, n, 0U, k);
+    return implementation->stream_ietf_xor_ic(c, m, mlen, n, 0U, k);
+}
+
+int
+_crypto_stream_chacha20_pick_best_implementation(void)
+{
+    implementation = &crypto_stream_chacha20_ref_implementation;
+#if (defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) && defined(__GNUC__))
+    if (sodium_runtime_has_ssse3()) {
+        implementation = &crypto_stream_chacha20_vec_implementation;
+    }
+#endif
+    return 0;
 }

data/vendor/libsodium/src/libsodium/crypto_stream/chacha20/stream_chacha20.h

@@ -0,0 +1,22 @@
+
+#ifndef stream_chacha20_H
+#define stream_chacha20_H
+
+#include <stdint.h>
+
+typedef struct crypto_stream_chacha20_implementation {
+    int (*stream)(unsigned char *c, unsigned long long clen,
+                  const unsigned char *n, const unsigned char *k);
+    int (*stream_ietf)(unsigned char *c, unsigned long long clen,
+                      const unsigned char *n, const unsigned char *k);
+    int (*stream_xor_ic)(unsigned char *c, const unsigned char *m,
+                         unsigned long long mlen,
+                         const unsigned char *n, uint64_t ic,
+                         const unsigned char *k);
+    int (*stream_ietf_xor_ic)(unsigned char *c, const unsigned char *m,
+                              unsigned long long mlen,
+                              const unsigned char *n, uint32_t ic,
+                              const unsigned char *k);
+} crypto_stream_chacha20_implementation;
+
+#endif

data/vendor/libsodium/src/libsodium/crypto_stream/chacha20/vec/stream_chacha20_vec.c

@@ -0,0 +1,329 @@
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "export.h"
+#include "utils.h"
+#include "crypto_stream_chacha20.h"
+#include "stream_chacha20_vec.h"
+#include "../stream_chacha20.h"
+
+#if (defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) && defined(__GNUC__))
+
+#pragma GCC target("sse2")
+#pragma GCC target("ssse3")
+
+#define CHACHA_RNDS 20
+
+typedef unsigned int vec __attribute__((vector_size(16)));
+
+#include <emmintrin.h>
+#include <tmmintrin.h>
+
+# if __clang__
+#  define VBPI 4
+# else
+#  define VBPI 3
+# endif
+# define ONE (vec) _mm_set_epi32(0, 0, 0, 1)
+# define LOAD(m) (vec) _mm_loadu_si128((const __m128i *) (const void *) (m))
+# define LOAD_ALIGNED(m) (vec) _mm_load_si128((const __m128i *) (const void *) (m))
+# define STORE(m, r) _mm_storeu_si128((__m128i *) (void *) (m), (__m128i) (r))
+# define ROTV1(x) (vec) _mm_shuffle_epi32((__m128i)x, _MM_SHUFFLE(0, 3, 2, 1))
+# define ROTV2(x) (vec) _mm_shuffle_epi32((__m128i)x, _MM_SHUFFLE(1, 0, 3, 2))
+# define ROTV3(x) (vec) _mm_shuffle_epi32((__m128i)x, _MM_SHUFFLE(2, 1, 0, 3))
+# define ROTW7(x) \
+    (vec)(_mm_slli_epi32((__m128i)x, 7) ^ _mm_srli_epi32((__m128i)x, 25))
+# define ROTW12(x) \
+    (vec)(_mm_slli_epi32((__m128i)x, 12) ^ _mm_srli_epi32((__m128i)x, 20))
+# define ROTW8(x) \
+    (vec)(_mm_slli_epi32((__m128i)x, 8) ^ _mm_srli_epi32((__m128i)x, 24))
+#define ROTW16(x) \
+    (vec)(_mm_slli_epi32((__m128i)x, 16) ^ _mm_srli_epi32((__m128i)x, 16))
+
+#ifndef REVV_BE
+# define REVV_BE(x) (x)
+#endif
+
+#define BPI (VBPI + 0) /* Blocks computed per loop iteration   */
+
+#define DQROUND_VECTORS(a, b, c, d) \
+    a += b;                         \
+    d ^= a;                         \
+    d = ROTW16(d);                  \
+    c += d;                         \
+    b ^= c;                         \
+    b = ROTW12(b);                  \
+    a += b;                         \
+    d ^= a;                         \
+    d = ROTW8(d);                   \
+    c += d;                         \
+    b ^= c;                         \
+    b = ROTW7(b);                   \
+    b = ROTV1(b);                   \
+    c = ROTV2(c);                   \
+    d = ROTV3(d);                   \
+    a += b;                         \
+    d ^= a;                         \
+    d = ROTW16(d);                  \
+    c += d;                         \
+    b ^= c;                         \
+    b = ROTW12(b);                  \
+    a += b;                         \
+    d ^= a;                         \
+    d = ROTW8(d);                   \
+    c += d;                         \
+    b ^= c;                         \
+    b = ROTW7(b);                   \
+    b = ROTV3(b);                   \
+    c = ROTV2(c);                   \
+    d = ROTV1(d);
+
+#define WRITE_XOR(in, op, d, v0, v1, v2, v3)           \
+    STORE(op + d + 0, LOAD(in + d + 0) ^ REVV_BE(v0)); \
+    STORE(op + d + 4, LOAD(in + d + 4) ^ REVV_BE(v1)); \
+    STORE(op + d + 8, LOAD(in + d + 8) ^ REVV_BE(v2)); \
+    STORE(op + d + 12, LOAD(in + d + 12) ^ REVV_BE(v3));
+
+struct chacha_ctx {
+    vec s1;
+    vec s2;
+    vec s3;
+};
+
+typedef struct chacha_ctx chacha_ctx;
+
+static void
+chacha_ivsetup(chacha_ctx *ctx, const uint8_t *iv, uint64_t ic)
+{
+    const vec s3 = {
+        (uint32_t) ic,
+        (uint32_t) (ic >> 32),
+        ((const uint32_t *) (const void *) iv)[0],
+        ((const uint32_t *) (const void *) iv)[1]
+    };
+    ctx->s3 = s3;
+}
+
+static void
+chacha_ietf_ivsetup(chacha_ctx *ctx, const uint8_t *iv, uint32_t ic)
+{
+    const vec s3 = {
+        ic,
+        ((const uint32_t *) (const void *) iv)[0],
+        ((const uint32_t *) (const void *) iv)[1],
+        ((const uint32_t *) (const void *) iv)[2]
+    };
+    ctx->s3 = s3;
+}
+
+static void
+chacha_keysetup(chacha_ctx *ctx, const uint8_t *k)
+{
+    ctx->s1 = LOAD(k);
+    ctx->s2 = LOAD(k + 16);
+}
+
+static void
+chacha_encrypt_bytes(chacha_ctx *ctx, const uint8_t *in, uint8_t *out,
+                     unsigned long long inlen)
+{
+    CRYPTO_ALIGN(16) unsigned chacha_const[]
+        = { 0x61707865, 0x3320646E, 0x79622D32, 0x6B206574 };
+    uint32_t           *op = (uint32_t *) (void *) out;
+    const uint32_t     *ip = (const uint32_t *) (const void *) in;
+    vec                 s0, s1, s2, s3;
+    unsigned long long  iters;
+    unsigned long long  i;
+
+    if (inlen > 64ULL * (1ULL << 32) - 64ULL) {
+        abort();
+    }
+    s0 = LOAD_ALIGNED(chacha_const);
+    s1 = ctx->s1;
+    s2 = ctx->s2;
+    s3 = ctx->s3;
+
+    for (iters = 0; iters < inlen / (BPI * 64); iters++) {
+#if VBPI > 2
+        vec v8, v9, v10, v11;
+#endif
+#if VBPI > 3
+        vec v12, v13, v14, v15;
+#endif
+        vec v0, v1, v2, v3, v4, v5, v6, v7;
+        v4 = v0 = s0;
+        v5 = v1 = s1;
+        v6 = v2 = s2;
+        v3 = s3;
+        v7 = v3 + ONE;
+#if VBPI > 2
+        v8 = v4;
+        v9 = v5;
+        v10 = v6;
+        v11 = v7 + ONE;
+#endif
+#if VBPI > 3
+        v12 = v8;
+        v13 = v9;
+        v14 = v10;
+        v15 = v11 + ONE;
+#endif
+        for (i = CHACHA_RNDS / 2; i; i--) {
+            DQROUND_VECTORS(v0, v1, v2, v3)
+            DQROUND_VECTORS(v4, v5, v6, v7)
+#if VBPI > 2
+            DQROUND_VECTORS(v8, v9, v10, v11)
+#endif
+#if VBPI > 3
+            DQROUND_VECTORS(v12, v13, v14, v15)
+#endif
+        }
+
+        WRITE_XOR(ip, op, 0, v0 + s0, v1 + s1, v2 + s2, v3 + s3)
+        s3 += ONE;
+        WRITE_XOR(ip, op, 16, v4 + s0, v5 + s1, v6 + s2, v7 + s3)
+        s3 += ONE;
+#if VBPI > 2
+        WRITE_XOR(ip, op, 32, v8 + s0, v9 + s1, v10 + s2, v11 + s3)
+        s3 += ONE;
+#endif
+#if VBPI > 3
+        WRITE_XOR(ip, op, 48, v12 + s0, v13 + s1, v14 + s2, v15 + s3)
+        s3 += ONE;
+#endif
+        ip += VBPI * 16;
+        op += VBPI * 16;
+    }
+
+    for (iters = inlen % (BPI * 64) / 64; iters != 0; iters--) {
+        vec v0 = s0, v1 = s1, v2 = s2, v3 = s3;
+        for (i = CHACHA_RNDS / 2; i; i--) {
+            DQROUND_VECTORS(v0, v1, v2, v3);
+        }
+        WRITE_XOR(ip, op, 0, v0 + s0, v1 + s1, v2 + s2, v3 + s3)
+        s3 += ONE;
+        ip += 16;
+        op += 16;
+    }
+
+    inlen = inlen % 64;
+    if (inlen) {
+        CRYPTO_ALIGN(16) vec buf[4];
+        vec v0, v1, v2, v3;
+        v0 = s0;
+        v1 = s1;
+        v2 = s2;
+        v3 = s3;
+        for (i = CHACHA_RNDS / 2; i; i--) {
+            DQROUND_VECTORS(v0, v1, v2, v3);
+        }
+
+        if (inlen >= 16) {
+            STORE(op + 0, LOAD(ip + 0) ^ REVV_BE(v0 + s0));
+            if (inlen >= 32) {
+                STORE(op + 4, LOAD(ip + 4) ^ REVV_BE(v1 + s1));
+                if (inlen >= 48) {
+                    STORE(op + 8, LOAD(ip + 8) ^ REVV_BE(v2 + s2));
+                    buf[3] = REVV_BE(v3 + s3);
+                } else {
+                    buf[2] = REVV_BE(v2 + s2);
+                }
+            } else {
+                buf[1] = REVV_BE(v1 + s1);
+            }
+        } else {
+            buf[0] = REVV_BE(v0 + s0);
+        }
+        for (i = inlen & ~15ULL; i < inlen; i++) {
+            ((char *) op)[i] = ((const char *) ip)[i] ^ ((char *) buf)[i];
+        }
+    }
+}
+
+static int
+stream_vec(unsigned char *c, unsigned long long clen,
+           const unsigned char *n, const unsigned char *k)
+{
+    struct chacha_ctx ctx;
+
+    if (!clen) {
+        return 0;
+    }
+    (void) sizeof(int[crypto_stream_chacha20_KEYBYTES == 256 / 8 ? 1 : -1]);
+    chacha_keysetup(&ctx, k);
+    chacha_ivsetup(&ctx, n, 0ULL);
+    memset(c, 0, clen);
+    chacha_encrypt_bytes(&ctx, c, c, clen);
+    sodium_memzero(&ctx, sizeof ctx);
+
+    return 0;
+}
+
+static int
+stream_ietf_vec(unsigned char *c, unsigned long long clen,
+                const unsigned char *n, const unsigned char *k)
+{
+    struct chacha_ctx ctx;
+
+    if (!clen) {
+        return 0;
+    }
+    (void) sizeof(int[crypto_stream_chacha20_KEYBYTES == 256 / 8 ? 1 : -1]);
+    chacha_keysetup(&ctx, k);
+    chacha_ietf_ivsetup(&ctx, n, 0ULL);
+    memset(c, 0, clen);
+    chacha_encrypt_bytes(&ctx, c, c, clen);
+    sodium_memzero(&ctx, sizeof ctx);
+
+    return 0;
+}
+
+static int
+stream_vec_xor_ic(unsigned char *c, const unsigned char *m,
+                  unsigned long long mlen,
+                  const unsigned char *n, uint64_t ic,
+                  const unsigned char *k)
+{
+    struct chacha_ctx ctx;
+
+    if (!mlen) {
+        return 0;
+    }
+    chacha_keysetup(&ctx, k);
+    chacha_ivsetup(&ctx, n, ic);
+    chacha_encrypt_bytes(&ctx, m, c, mlen);
+    sodium_memzero(&ctx, sizeof ctx);
+
+    return 0;
+}
+
+static int
+stream_ietf_vec_xor_ic(unsigned char *c, const unsigned char *m,
+                       unsigned long long mlen,
+                       const unsigned char *n, uint32_t ic,
+                       const unsigned char *k)
+{
+    struct chacha_ctx ctx;
+
+    if (!mlen) {
+        return 0;
+    }
+    chacha_keysetup(&ctx, k);
+    chacha_ietf_ivsetup(&ctx, n, ic);
+    chacha_encrypt_bytes(&ctx, m, c, mlen);
+    sodium_memzero(&ctx, sizeof ctx);
+
+    return 0;
+}
+
+struct crypto_stream_chacha20_implementation
+crypto_stream_chacha20_vec_implementation = {
+    SODIUM_C99(.stream =) stream_vec,
+    SODIUM_C99(.stream_ietf =) stream_ietf_vec,
+    SODIUM_C99(.stream_xor_ic =) stream_vec_xor_ic,
+    SODIUM_C99(.stream_ietf_xor_ic =) stream_ietf_vec_xor_ic
+};
+
+#endif