libmongocrypt-helper 1.7.4.0.1002 → 1.11.0.0.1001

data/ext/libmongocrypt/libmongocrypt/etc/fle2_crypto.py

@@ -8,6 +8,7 @@ MAC_KEY_LENGTH = 32
 HMAC_SHA256_TAG_LENGTH = 32
 IV_LENGTH = 16
 DEK_LENGTH = 96
+BLOCK_LENGTH = 16
 
 def _hmacsha256 (Km, input):
     assert (len(Km) == MAC_KEY_LENGTH)
@@ -25,81 +26,92 @@ class DEK ():
         self.Km = bytesIn[32:64]
         self.TokenKey = bytesIn[64:96]
 
-def fle2_encrypt (M, Ke, IV):
+def _fle2_encrypt (IV, Ke, M, mode, Km = None, AD = None):
     """
-    Compute 
-    S = AES-CTR.Enc(Ke, IV, M)
-
-    Output 
-    C = IV || S
+    Generalized encrypt vector create.
+    S = AES-{mode}.Enc(Ke, IV, M)
+    if Km is not None:
+        T = HMAC/SHA-256(Km, AD || IV || S)
+    C = IV || S || T
     """
-    assert (len(Ke) == ENCRYPTION_KEY_LENGTH)
-    assert (len(IV) == IV_LENGTH)
+    assert len(Ke) == ENCRYPTION_KEY_LENGTH
+    assert len(IV) == IV_LENGTH
+    assert (mode == 'CTR') or (mode == 'CBC')
+    modeObj = modes.CTR(IV) if mode == 'CTR' else modes.CBC(IV)
 
-    # S = AES-CTR.Enc(Ke, IV, M)
-    cipher = Cipher(algorithms.AES(Ke), modes.CTR(IV))
+    # S = AES-{mode}.Enc(Ke, IV, M)
+    cipher = Cipher(algorithms.AES(Ke), modeObj)
     encryptor = cipher.encryptor()
+    if mode == 'CBC':
+        # PKCS#7
+        padding_len = BLOCK_LENGTH - (len(M) % BLOCK_LENGTH)
+        M = M + (padding_len.to_bytes(1, 'big') * padding_len)
     S = encryptor.update(M) + encryptor.finalize()
 
-    C = IV + S
+    if Km is not None:
+        assert AD is not None
+        assert len(Km) == MAC_KEY_LENGTH
+        # T = HMAC-SHA256(Km, AD || IV || S)
+        T = _hmacsha256 (Km, AD + IV + S)
+    else:
+        assert AD is None
+        T = b''
+
+    # C = IV + S + T
+    C = IV + S + T
     return C
 
-def fle2_decrypt (C, Ke):
-    assert (len(Ke) == ENCRYPTION_KEY_LENGTH)
-    assert (len(C) > IV_LENGTH)
 
-    IV = C[0:IV_LENGTH]
-    # S = AES-CTR.Enc(Ke, IV, M)
-    cipher = Cipher(algorithms.AES(Ke), modes.CTR(IV))
-    encryptor = cipher.decryptor()
-    M = encryptor.update(C[IV_LENGTH:]) + encryptor.finalize()
-    return M
+def fle2_encrypt (M, Ke, IV):
+    """ AES-256-CTR/NONE """
+    return _fle2_encrypt (IV, Ke, M, 'CTR')
 
 def fle2aead_encrypt(M, Ke, IV, Km, AD):
-    """
-    Do FLE 2 AEAD encryption.
-    See [AEAD with CTR](https://docs.google.com/document/d/1eCU7R8Kjr-mdyz6eKvhNIDVmhyYQcAaLtTfHeK7a_vE/edit#heading=h.35kjadvlcbty)
-    See [aead_encryption_fle2_test_vectors.sh](https://github.com/mongodb/mongo/blob/ecc66915ac757cbeaa7c40eb443d7ec7bffcb80a/src/mongo/crypto/scripts/aead_encryption_fle2_test_vectors.sh#L15) for how server team is generating this.
-    """
-    assert (len(Ke) == ENCRYPTION_KEY_LENGTH)
-    assert (len(IV) == IV_LENGTH)
-    assert (len(Km) == MAC_KEY_LENGTH)
+    """ AES-256-CTR/SHA-256 """
+    return _fle2_encrypt (IV, Ke, M, 'CTR', Km, AD)
 
-    # S = AES-CTR.Enc(Ke, IV, M)
-    cipher = Cipher(algorithms.AES(Ke), modes.CTR(IV))
-    encryptor = cipher.encryptor()
-    S = encryptor.update(M) + encryptor.finalize()
+def fle2v2_aead_encrypt(M, Ke, IV, Km, AD):
+    """ AES-256-CBC/SHA-256 """
+    return _fle2_encrypt (IV, Ke, M, 'CBC', Km, AD)
 
-    # T = HMAC-SHA256(Km, AD || IV || S)
-    T = _hmacsha256 (Km, AD + IV + S)
+def _fle2_decrypt (C, Ke, mode, Km = None, AD = None):
+    assert (len(Ke) == ENCRYPTION_KEY_LENGTH)
 
+    Tlen = 0 if Km is None else HMAC_SHA256_TAG_LENGTH;
+    assert (len(C) > (IV_LENGTH + Tlen))
     # C = IV || S || T
-    C = IV + S + T
-    return C
+    IV = C[0:IV_LENGTH]
+    S = C[IV_LENGTH:-Tlen]
 
+    if Km is not None:
+        T = C[-Tlen:]
+        assert T == _hmacsha256 (Km, AD + IV + S)
 
-def fle2aead_decrypt(C, Km, AD, Ke):
-    assert (len(Ke) == ENCRYPTION_KEY_LENGTH)
-    assert (len(C) > HMAC_SHA256_TAG_LENGTH + IV_LENGTH)
-    assert (len(Km) == MAC_KEY_LENGTH)
+    assert (mode == 'CTR') or (mode == 'CBC')
+    modeObj = modes.CTR(IV) if mode == 'CTR' else modes.CBC(IV)
 
-    # Parse C as IV || S || T
-    IV = C[0:IV_LENGTH]
-    S = C[IV_LENGTH:-HMAC_SHA256_TAG_LENGTH]
-    T = C[-HMAC_SHA256_TAG_LENGTH:]
+    # M = AES-{mode}.Dec(Ke, IV, S)
+    cipher = Cipher(algorithms.AES(Ke), modeObj)
+    encryptor = cipher.decryptor()
+    if mode == 'CBC':
+        # PKCS#7
+        padding_len = ord(S[-1:])
+        S = S[-padding_len:]
+    M = encryptor.update(S) + encryptor.finalize()
 
-    # Compute T' = HMAC-SHA256(Km, AD || IV || S)
-    Tp = _hmacsha256 (Km, AD + IV + S)
-    if Tp != T:
-        raise Exception("decryption error")
+    return M
 
-    # Else compute and output M = AES-CTR.Dec(Ke, S)
-    cipher = Cipher(algorithms.AES(Ke), modes.CTR(IV))
-    decryptor = cipher.decryptor()
-    M = decryptor.update(S) + decryptor.finalize()
+def fle2_decrypt (C, Ke):
+    """AES-256-CTR/NONE"""
+    return _fle2_decrypt (C, Ke, 'CTR')
 
-    return M
+def fle2aead_decrypt(C, Km, AD, Ke):
+    """AES-256-CTR/SHA-256"""
+    return _fle2_decrypt (C, Ke, 'CTR', Km, AD)
 
+def fle2v2_aead_decrypt(C, Km, AD, Ke):
+    """AES-256-CBC/SHA-256"""
+    return _fle2_decrypt (C, Ke, 'CBC', Km, AD)
 
 def ServerDataEncryptionLevel1Token (rootKey):
-    return _hmacsha256 (rootKey, struct.pack("<Q", 3))
+    return _hmacsha256 (rootKey, struct.pack("<Q", 3))

data/ext/libmongocrypt/libmongocrypt/etc/fle2_generate_tests.py

@@ -1,34 +1,25 @@
-# fle2_generate_tests.py is used to generate the test file: ./test/data/fle2.cstructs
+# fle2_generate_tests.py is used to generate the test file: ./test/data/roundtrip/fle2-generated.json
+import json
 import os
 import fle2_crypto
 
 # Generate test cases of various and plaintext (M) lengths.
 M_lens = [1, 16, 64, 100]
-
-tmpl = """{{
-    .testname = "{testname}",
-    .iv = "{iv}",
-    .key = "{key}",
-    .plaintext = "{plaintext}",
-    .ciphertext = "{ciphertext}",
-    .bytes_written_expected = {bytes_written_expected}
-}},"""
-
-print("/* This file is generated by ./etc/fle2_generate_tests.py. */")
+out = []
 for M_len in M_lens:
     M = os.urandom(M_len)
     IV = os.urandom(16)
     Ke = os.urandom(32)
     C = fle2_crypto.fle2_encrypt(M=M, Ke=Ke, IV=IV)
 
-    key = Ke
-    args = {
-        "testname": "generated test. M length={}".format (M_len),
-        "iv": IV.hex(),
-        "key": key.hex(),
-        "plaintext": M.hex(),
-        "ciphertext": C.hex(),
-        "bytes_written_expected": len(C)
-    }
-    testcase = tmpl.format(**args)
-    print(testcase)
+    out.append({
+      'name': "generated test. M length={}".format (M_len),
+      'origin': "etc/fle2_generate_tests.py",
+      'algo': 'AES-256-CTR/NONE',
+      'iv': IV.hex(),
+      'key': Ke.hex(),
+      'plaintext': M.hex(),
+      'ciphertext': C.hex()
+   })
+
+print(json.JSONEncoder(indent=3).encode(out))

data/ext/libmongocrypt/libmongocrypt/etc/fle2v2_aead_generate_tests.py

@@ -0,0 +1,32 @@
+# fle2v2_aead_generate_tests.py is used to generate the test file: ./test/data/roundtrip/fle2v2-aead-generated.json
+import json
+import os
+import itertools
+import fle2_crypto
+
+# Generate test cases by taking the cross-product of AD (associated data) and plaintext (M) lengths.
+AD_lens = [5, 10, 20]
+M_lens = [1, 16, 64, 100]
+out = []
+for (AD_len, M_len) in itertools.product(AD_lens, M_lens):
+    M = os.urandom(M_len)
+    AD = os.urandom(AD_len)
+    IV = os.urandom(16)
+    Ke = os.urandom(32)
+    Km = os.urandom(32)
+    C = fle2_crypto.fle2v2_aead_encrypt(M=M, Ke=Ke, IV=IV, Km=Km, AD=AD)
+
+    # Create the 96 byte data encryption key. The last 32 are unused.
+    key = Ke + Km + (b"\x00" * 32)
+    out.append({
+      'name': "generated test. AD length={}, M length={}".format(AD_len, M_len),
+      'origin': "etc/fle2v2_aead_generate_tests.py",
+      'algo': 'AES-256-CBC/SHA-256',
+      'iv': IV.hex(),
+      'aad': AD.hex(),
+      'key': key.hex(),
+      'plaintext': M.hex(),
+      'ciphertext': C.hex()
+   })
+
+print(json.JSONEncoder(indent=3).encode(out))

data/ext/libmongocrypt/libmongocrypt/etc/format-all.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+. "$(dirname "${BASH_SOURCE[0]}")/../.evergreen/init.sh"
+
+files="$(find "$LIBMONGOCRYPT_DIR/src" "$LIBMONGOCRYPT_DIR/test" -type f -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp')"
+# shellcheck disable=SC2206
+IFS=$'\n' files=($files)
+bash "$LIBMONGOCRYPT_DIR/etc/format.sh" \
+    --style="file:$LIBMONGOCRYPT_DIR/.clang-format" \
+    -i \
+    "$@" \
+    -- "${files[@]:?}"

data/ext/libmongocrypt/libmongocrypt/etc/format.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+. "$(dirname "${BASH_SOURCE[0]}")/../.evergreen/init.sh"
+
+: "${CLANG_FORMAT_VERSION:=15.0.7}"
+export PIPX_HOME="${BUILD_CACHE_DIR}/pipx"
+
+if ! run_python -c ''; then
+  fail "No Python found?"
+fi
+
+# Check that we have a pipx of the proper version:
+run_python -c 'import pkg_resources; pkg_resources.require("pipx>=0.17.0,<2.0")'
+
+# Give default clang-format an empty string on stdin if there are no inputs files
+printf '' | run_python -m pipx run "clang-format==${CLANG_FORMAT_VERSION:?}" "$@"

data/ext/libmongocrypt/libmongocrypt/etc/install-package.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env sh
+
+# Install the named packages using the platform's package manager.
+# This script "just does the right thing" to install a package as a step in a
+#   caching container build, as a bare "pkg-tool install" might be insufficient
+#   or subtly broken, and we only really care about installing a list of packages
+#
+# Usage: __install [pkgs...]
+
+set -eu
+
+if test -f /etc/debian_version ; then
+    # We *must* do a repo update as part of the install step in a single cache
+    # layer, as a cached apt-get update can become stale while a later
+    # apt-get install is invalidated
+    apt-get -y update
+    # DEBIAN_FRONTEND suppresses all interactivity. For some reason some
+    # packages like to prompt as part of their config, even if we're not on
+    # a TTY.
+    env DEBIAN_FRONTEND=noninteractive \
+        apt-get -y install -- "$@"
+elif test -f /etc/redhat-release || grep 'ID="amzn"' /etc/os-release >/dev/null 1>&2; then
+    if test -f /usr/bin/dnf; then
+        # 'dnf' will "do the right thing"
+        dnf install -y -- "$@"
+    elif test -f /usr/bin/yum; then
+        yum install -y -- "$@"
+        # 'yum' happily ignores missing packages. Use 'rpm -q' to check that
+        # everything we requested actually got installed.
+        if ! rpm -q -- "$@"; then
+            echo "__install: Failing because one or more packages requested are not available"
+            exit 1
+        fi
+    else
+        echo "No package manager here?" 1>&2
+        exit 1
+    fi
+elif test -f /etc/SuSE-brand \
+     || (test -f /etc/os-release && grep "opensuse" /etc/os-release); then
+    zypper --non-interactive install "$@"
+elif test -f /etc/arch-release; then
+    pacman -Sy --noconfirm -- "$@"
+elif test -f /etc/alpine-release; then
+    apk add -- "$@"
+else
+    echo "We don't know how to manage packages on this system" 1>&2
+    exit 1
+fi

data/ext/libmongocrypt/libmongocrypt/etc/libbson-remove-GCC-diagnostic-pragma.patch

@@ -0,0 +1,27 @@
+diff --git a/src/common/bson-dsl.h b/src/common/bson-dsl.h
+index 58a14e28e..3341e603f 100644
+--- a/src/common/bson-dsl.h
++++ b/src/common/bson-dsl.h
+@@ -29,6 +29,13 @@ enum {
+    BSON_IF_WINDOWS (__declspec (selectany)) \
+    BSON_IF_POSIX (__attribute__ ((weak)))
+ 
++#if defined(__GNUC__) && !defined(__clang__) && ((__GNUC__ < 4) || (__GNUC__ == 4 && __GNUC_MINOR__ < 6))
++// Using GCC < 4.6
++// Do not define `GCC diagnostic` pragma for GCC < 4.6.
++#define _bsonDSL_disableWarnings() ((void) 0)
++#define _bsonDSL_restoreWarnings() ((void) 0)
++#else
++// Not using GCC < 4.6
+ #ifdef __GNUC__
+ // GCC has a bug handling pragma statements that disable warnings within complex
+ // nested macro expansions. If we're GCC, just disable -Wshadow outright:
+@@ -47,7 +54,7 @@ BSON_IF_GNU_LIKE (_Pragma ("GCC diagnostic ignored \"-Wshadow\""))
+       BSON_IF_GNU_LIKE (_Pragma ("GCC diagnostic pop");) \
+    } else                                                \
+       ((void) 0)
+-
++#endif
+ /**
+  * @brief Parse the given BSON document.
+  *

data/ext/libmongocrypt/libmongocrypt/etc/mongo-inteldfp-alpine-arm-fix.patch

@@ -0,0 +1,17 @@
+diff --git a/cmake-build/_deps/intel_dfp-src/LIBRARY/src/bid_functions.h b/cmake-build/_deps/intel_dfp-src/LIBRARY/src/bid_functions.h
+index 7042eed..e55f59e 100755
+--- a/cmake-build/_deps/intel_dfp-src/LIBRARY/src/bid_functions.h
++++ b/cmake-build/_deps/intel_dfp-src/LIBRARY/src/bid_functions.h
+@@ -41,12 +41,6 @@
+ #endif
+ #include <ctype.h>
+ 
+-// Fix system header issue on Sun solaris and define required type by ourselves
+-#if !defined(_WCHAR_T) && !defined(_WCHAR_T_DEFINED) && !defined(__QNX__) && !defined(__cplusplus)
+-typedef int   wchar_t;
+-#endif
+-
+-
+ #ifdef IN_LIBGCC2
+ // When we are built as the part of the gcc runtime library, libgcc,
+ // we will use gcc types defined in bid_gcc_intrinsics.h.

data/ext/libmongocrypt/libmongocrypt/etc/mongo-inteldfp-libmongocrypt-pr-625.patch

@@ -0,0 +1,13 @@
+diff --git a/src/third_party/IntelRDFPMathLib20U1/LIBRARY/float128/architecture.h b/src/third_party/IntelRDFPMathLib20U1/LIBRARY/float128/architecture.h
+index ef8a5b7..98d86c1 100644
+--- a/src/third_party/IntelRDFPMathLib20U1/LIBRARY/float128/architecture.h
++++ b/src/third_party/IntelRDFPMathLib20U1/LIBRARY/float128/architecture.h
+@@ -39,7 +39,7 @@
+ #   define merced
+ #endif
+ 
+-#if defined(ct) || defined(efi2)
++#if defined(ct) || defined(efi2) || defined(__x86_64__)
+ #   undef  _M_AMD64
+ #   define _M_AMD64
+ #endif