libmongocrypt-helper 1.12.0.0.1001 → 1.14.0.0.1001

data/ext/libmongocrypt/libmongocrypt/src/mc-text-search-str-encode.c

@@ -0,0 +1,324 @@
+/*
+ * Copyright 2024-present MongoDB, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "mc-fle2-encryption-placeholder-private.h"
+#include "mc-str-encode-string-sets-private.h"
+#include "mc-text-search-str-encode-private.h"
+#include "mongocrypt-buffer-private.h"
+#include "mongocrypt-crypto-private.h"
+#include "mongocrypt.h"
+#include "unicode/fold.h"
+#include <bson/bson.h>
+#include <stdint.h>
+
+// 16MiB - maximum length in bytes of a string to be encoded.
+#define MAX_ENCODE_BYTE_LEN 16777216
+// Number of bytes which are added to the base string before encryption.
+#define OVERHEAD_BYTES 5
+
+static mc_affix_set_t *generate_prefix_or_suffix_tree(const mc_utf8_string_with_bad_char_t *base_str,
+                                                      uint32_t unfolded_byte_len,
+                                                      uint32_t lb,
+                                                      uint32_t ub,
+                                                      bool is_prefix) {
+    BSON_ASSERT_PARAM(base_str);
+    // We encrypt (unfolded string + 5 bytes of extra BSON info) with a 16-byte block cipher.
+    uint32_t encrypted_len = 16 * (uint32_t)((unfolded_byte_len + OVERHEAD_BYTES + 15) / 16);
+    // Max len of a string that has this encrypted len.
+    uint32_t padded_len = encrypted_len - OVERHEAD_BYTES;
+    if (padded_len < lb) {
+        // No valid substrings, return empty tree
+        return NULL;
+    }
+
+    // Total number of substrings
+    uint32_t msize = BSON_MIN(padded_len, ub) - lb + 1;
+    uint32_t folded_codepoint_len = base_str->codepoint_len - 1; // remove one codepoint for 0xFF
+    uint32_t real_max_len = BSON_MIN(folded_codepoint_len, ub);
+    // Number of actual substrings, excluding padding
+    uint32_t real_substrings = real_max_len >= lb ? real_max_len - lb + 1 : 0;
+    // If real_substrings and msize differ, we need to insert padding, so allocate one extra slot.
+    uint32_t set_size = real_substrings == msize ? real_substrings : real_substrings + 1;
+    mc_affix_set_t *set = mc_affix_set_new(base_str, set_size);
+    uint32_t n_inserted = 0;
+    for (uint32_t i = lb; i < real_max_len + 1; i++, n_inserted++) {
+        if (is_prefix) {
+            // [0, lb), [0, lb + 1), ..., [0, min(len, ub))
+            BSON_ASSERT(mc_affix_set_insert(set, 0, i));
+        } else {
+            // [len - lb, len), [len - lb - 1, len), ..., [max(0, len - ub), len)
+            BSON_ASSERT(mc_affix_set_insert(set, folded_codepoint_len - i, folded_codepoint_len));
+        }
+    }
+    if (msize != real_substrings) {
+        // Insert padding to get to msize
+        BSON_ASSERT(mc_affix_set_insert_base_string(set, msize - real_substrings));
+        n_inserted++;
+    }
+    BSON_ASSERT(n_inserted == set_size);
+    return set;
+}
+
+static mc_affix_set_t *generate_suffix_tree(const mc_utf8_string_with_bad_char_t *base_str,
+                                            uint32_t unfolded_byte_len,
+                                            const mc_FLE2SuffixInsertSpec_t *spec) {
+    BSON_ASSERT_PARAM(base_str);
+    BSON_ASSERT_PARAM(spec);
+    return generate_prefix_or_suffix_tree(base_str, unfolded_byte_len, spec->lb, spec->ub, false);
+}
+
+static mc_affix_set_t *generate_prefix_tree(const mc_utf8_string_with_bad_char_t *base_str,
+                                            uint32_t unfolded_byte_len,
+                                            const mc_FLE2PrefixInsertSpec_t *spec) {
+    BSON_ASSERT_PARAM(base_str);
+    BSON_ASSERT_PARAM(spec);
+    return generate_prefix_or_suffix_tree(base_str, unfolded_byte_len, spec->lb, spec->ub, true);
+}
+
+static uint32_t calc_number_of_substrings(uint32_t strlen, uint32_t lb, uint32_t ub) {
+    // There are len - i + 1 substrings of length i in a length len string.
+    // Therefore, the total number of substrings with length between lb and ub
+    // is the sum of the integers inclusive between A = len - ub + 1 and B = len - lb + 1,
+    // A <= B. This has a closed form: (A + B)(B - A + 1)/2.
+    if (lb > strlen) {
+        return 0;
+    }
+    uint32_t largest_substr = BSON_MIN(strlen, ub);
+    uint32_t largest_substr_count = strlen - largest_substr + 1;
+    uint32_t smallest_substr_count = strlen - lb + 1;
+    return (largest_substr_count + smallest_substr_count) * (smallest_substr_count - largest_substr_count + 1) / 2;
+}
+
+static mc_substring_set_t *generate_substring_tree(const mc_utf8_string_with_bad_char_t *base_str,
+                                                   uint32_t unfolded_byte_len,
+                                                   const mc_FLE2SubstringInsertSpec_t *spec) {
+    BSON_ASSERT_PARAM(base_str);
+    BSON_ASSERT_PARAM(spec);
+    // We encrypt (unfolded string + 5 bytes of extra BSON info) with a 16-byte block cipher.
+    uint32_t encrypted_len = 16 * (uint32_t)((unfolded_byte_len + OVERHEAD_BYTES + 15) / 16);
+    // Max len of a string that has this encrypted len.
+    uint32_t padded_len = encrypted_len - OVERHEAD_BYTES;
+    if (padded_len < spec->lb) {
+        // No valid substrings, return empty tree
+        return NULL;
+    }
+
+    // If you are following along with the OST paper, a slightly different calculation of msize is used. The following
+    // justifies why that calculation and this calculation are equivalent.
+    // At this point, it is established that:
+    //     beta <= mlen
+    //     lb <= padded_len
+    //     lb <= ub <= mlen
+    //
+    // So, the following formula for msize in the OST paper:
+    //     maxkgram_1 = sum_(j=lb, ub, (mlen - j + 1))
+    //     maxkgram_2 = sum_(j=lb, min(ub, padded_len), (padded_len - j + 1))
+    //     msize      = min(maxkgram_1, maxkgram_2)
+    // can be simplified to:
+    //     msize      = sum_(j=lb, min(ub, padded_len), (min(mlen, padded_len) - j + 1))
+    //
+    // because if padded_len <= ub, then it follows that padded_len <= ub <= mlen, and so
+    //     maxkgram_1 = sum_(j=lb, ub, (mlen - j + 1))          # as above
+    //     maxkgram_2 = sum_(j=lb, padded_len, (padded_len - j + 1))    # less or equal to maxkgram_1
+    //     msize      = maxkgram_2
+    // and if padded_len > ub, then it follows that:
+    //     maxkgram_1 = sum_(j=lb, ub, (mlen - j + 1))          # as above
+    //     maxkgram_2 = sum_(j=lb, ub, (padded_len - j + 1))        # same sum bounds as maxkgram_1
+    //     msize      = sum_(j=lb, ub, (min(mlen, padded_len) - j + 1))
+    // in both cases, msize can be rewritten as:
+    //     msize      = sum_(j=lb, min(ub, padded_len), (min(mlen, padded_len) - j + 1))
+
+    uint32_t folded_codepoint_len = base_str->codepoint_len - 1;
+    // If mlen < padded_len, we only need to pad to mlen
+    padded_len = BSON_MIN(spec->mlen, padded_len);
+    // Total number of substrings -- i.e. the number of valid substrings IF the string spanned the full padded length
+    uint32_t msize = calc_number_of_substrings(padded_len, spec->lb, spec->ub);
+    uint32_t n_real_substrings = 0;
+    mc_substring_set_t *set = mc_substring_set_new(base_str);
+    // If folded len < LB, there are no real substrings, so we can skip (avoiding underflow via folded len - LB)
+    if (folded_codepoint_len >= spec->lb) {
+        for (uint32_t i = 0; i < folded_codepoint_len - spec->lb + 1; i++) {
+            for (uint32_t j = i + spec->lb; j < BSON_MIN(folded_codepoint_len, i + spec->ub) + 1; j++) {
+                // Only count successful, i.e. non-duplicate inserts
+                if (mc_substring_set_insert(set, i, j)) {
+                    n_real_substrings++;
+                }
+            }
+        }
+    }
+    if (msize != n_real_substrings) {
+        // Insert msize - n_real_substrings padding
+        BSON_ASSERT(msize > n_real_substrings);
+        mc_substring_set_increment_fake_string(set, msize - n_real_substrings);
+    }
+    return set;
+}
+
+static uint32_t mc_get_utf8_codepoint_length(const char *buf, uint32_t len) {
+    BSON_ASSERT_PARAM(buf);
+    const char *cur = buf;
+    const char *end = buf + len;
+    uint32_t codepoint_len = 0;
+    while (cur < end) {
+        cur = bson_utf8_next_char(cur);
+        codepoint_len++;
+    }
+    return codepoint_len;
+}
+
+mc_str_encode_sets_t *mc_text_search_str_encode(const mc_FLE2TextSearchInsertSpec_t *spec,
+                                                mongocrypt_status_t *status) {
+    BSON_ASSERT_PARAM(spec);
+    if (spec->len > MAX_ENCODE_BYTE_LEN) {
+        CLIENT_ERR("StrEncode: String passed in was too long: String was %" PRIu32 " bytes, but max is %d bytes",
+                   spec->len,
+                   MAX_ENCODE_BYTE_LEN);
+        return NULL;
+    }
+
+    if (!bson_utf8_validate(spec->v, spec->len, false /* allow_null */)) {
+        CLIENT_ERR("StrEncode: String passed in was not valid UTF-8");
+        return NULL;
+    }
+
+    mc_utf8_string_with_bad_char_t *base_string;
+    if (spec->casef || spec->diacf) {
+        char *folded_str;
+        size_t folded_str_bytes_len;
+        if (!unicode_fold(spec->v,
+                          spec->len,
+                          (spec->casef * kUnicodeFoldToLower) | (spec->diacf * kUnicodeFoldRemoveDiacritics),
+                          &folded_str,
+                          &folded_str_bytes_len,
+                          status)) {
+            return NULL;
+        }
+        base_string = mc_utf8_string_with_bad_char_from_buffer(folded_str, (uint32_t)folded_str_bytes_len);
+        bson_free(folded_str);
+    } else {
+        base_string = mc_utf8_string_with_bad_char_from_buffer(spec->v, spec->len);
+    }
+
+    mc_str_encode_sets_t *sets = bson_malloc0(sizeof(mc_str_encode_sets_t));
+    // Base string is the folded string plus the 0xFF character
+    sets->base_string = base_string;
+    if (spec->suffix.set) {
+        sets->suffix_set = generate_suffix_tree(sets->base_string, spec->len, &spec->suffix.value);
+    }
+    if (spec->prefix.set) {
+        sets->prefix_set = generate_prefix_tree(sets->base_string, spec->len, &spec->prefix.value);
+    }
+    if (spec->substr.set) {
+        uint32_t unfolded_codepoint_len = mc_get_utf8_codepoint_length(spec->v, spec->len);
+        if (unfolded_codepoint_len > spec->substr.value.mlen) {
+            CLIENT_ERR("StrEncode: String passed in was longer than the maximum length for substring indexing -- "
+                       "String len: %u, max len: %u",
+                       unfolded_codepoint_len,
+                       spec->substr.value.mlen);
+            mc_str_encode_sets_destroy(sets);
+            return NULL;
+        }
+        sets->substring_set = generate_substring_tree(sets->base_string, spec->len, &spec->substr.value);
+    }
+    // Exact string is always equal to the base string up until the bad character
+    _mongocrypt_buffer_from_data(&sets->exact, sets->base_string->buf.data, (uint32_t)sets->base_string->buf.len - 1);
+    return sets;
+}
+
+void mc_str_encode_sets_destroy(mc_str_encode_sets_t *sets) {
+    if (!sets) {
+        return;
+    }
+    mc_utf8_string_with_bad_char_destroy(sets->base_string);
+    mc_affix_set_destroy(sets->suffix_set);
+    mc_affix_set_destroy(sets->prefix_set);
+    mc_substring_set_destroy(sets->substring_set);
+    bson_free(sets);
+}
+
+bool mc_text_search_str_query(const mc_FLE2TextSearchInsertSpec_t *spec,
+                              _mongocrypt_buffer_t *out,
+                              mongocrypt_status_t *status) {
+    BSON_ASSERT_PARAM(spec);
+    BSON_ASSERT_PARAM(out);
+
+    if (spec->len > MAX_ENCODE_BYTE_LEN) {
+        CLIENT_ERR("StrQuery: String passed in was too long: String was %" PRIu32 " bytes, but max is %d bytes",
+                   spec->len,
+                   MAX_ENCODE_BYTE_LEN);
+        return false;
+    }
+
+    _mongocrypt_buffer_init(out);
+    if (!bson_utf8_validate(spec->v, spec->len, false /* allow_null */)) {
+        CLIENT_ERR("StrQuery: String passed in was not valid UTF-8");
+        return false;
+    }
+
+    uint32_t folded_codepoint_len = 0;
+
+    if (spec->casef || spec->diacf) {
+        char *folded_str;
+        size_t folded_str_bytes_len;
+        if (!unicode_fold(spec->v,
+                          spec->len,
+                          (spec->casef * kUnicodeFoldToLower) | (spec->diacf * kUnicodeFoldRemoveDiacritics),
+                          &folded_str,
+                          &folded_str_bytes_len,
+                          status)) {
+            return false;
+        }
+        _mongocrypt_buffer_copy_from_string_as_bson_value(out, folded_str, (int)folded_str_bytes_len);
+        folded_codepoint_len = mc_get_utf8_codepoint_length(folded_str, (uint32_t)folded_str_bytes_len);
+        bson_free(folded_str);
+    } else {
+        _mongocrypt_buffer_copy_from_string_as_bson_value(out, spec->v, (int)spec->len);
+        folded_codepoint_len = mc_get_utf8_codepoint_length(spec->v, spec->len);
+    }
+
+    if (spec->substr.set || spec->suffix.set || spec->prefix.set) {
+        uint32_t min = 0, max = 0;
+        if (spec->substr.set) {
+            min = spec->substr.value.lb;
+            max = spec->substr.value.ub;
+        } else if (spec->suffix.set) {
+            min = spec->suffix.value.lb;
+            max = spec->suffix.value.ub;
+        } else {
+            min = spec->prefix.value.lb;
+            max = spec->prefix.value.ub;
+        }
+        if (folded_codepoint_len == 0) {
+            CLIENT_ERR("StrQuery: string value cannot be empty for substring, suffix, or prefix queries");
+            return false;
+        }
+        if (folded_codepoint_len > max) {
+            CLIENT_ERR("StrQuery: string value was longer than the maximum query length "
+                       "for this field after folding -- folded codepoint len: %u, max query len: %u",
+                       folded_codepoint_len,
+                       max);
+            return false;
+        }
+        if (folded_codepoint_len < min) {
+            CLIENT_ERR("StrQuery: string value was shorter than the minimum query length "
+                       "for this field after folding -- folded codepoint len: %u, min query len: %u",
+                       folded_codepoint_len,
+                       min);
+            return false;
+        }
+    }
+    return true;
+}

data/ext/libmongocrypt/libmongocrypt/src/mc-tokens-private.h

@@ -73,6 +73,47 @@
  * Server-side:
  *      AnchorPaddingTokenKey    = HMAC(AnchorPaddingTokenRoot, 1)
  *      AnchorPaddingTokenValue  = HMAC(AnchorPaddingTokenRoot, 2)
+ * ----------------------------------------------------------------------------
+ * Added in Text Search:
+ *
+ * EDCTextExactToken = HMAC(EDCToken, 1)
+ * EDCTextSubstringToken = HMAC(EDCToken, 2)
+ * EDCTextSuffixToken = HMAC(EDCToken, 3)
+ * EDCTextPrefixToken = HMAC(EDCToken, 4)
+ *
+ * ESCTextExactToken = HMAC(ESCToken, 1)
+ * ESCTextSubstringToken = HMAC(ESCToken, 2)
+ * ESCTextSuffixToken = HMAC(ESCToken, 3)
+ * ESCTextPrefixToken = HMAC(ESCToken, 4)
+ *
+ * ServerTextExactToken = HMAC(ServerTokenDerivationLevel1Token, 1)
+ * ServerTextSubstringToken = HMAC(ServerTokenDerivationLevel1Token, 2)
+ * ServerTextSuffixToken = HMAC(ServerTokenDerivationLevel1Token, 3)
+ * ServerTextPrefixToken = HMAC(ServerTokenDerivationLevel1Token, 4)
+ *
+ * EDCTextExactDerivedFromDataTokenAndContentionFactorToken =
+ *     HMAC(HMAC(EDCTextExactToken, v), u)
+ * EDCTextSubstringDerivedFromDataTokenAndContentionFactorToken =
+ *     HMAC(HMAC(EDCTextSubstringToken, v), u)
+ * EDCTextSuffixDerivedFromDataTokenAndContentionFactorToken =
+ *     HMAC(HMAC(EDCTextSuffixToken, v), u)
+ * EDCTextPrefixDerivedFromDataTokenAndContentionFactorToken =
+ *     HMAC(HMAC(EDCTextPrefixToken, v), u)
+ *
+ * ESCTextExactDerivedFromDataTokenAndContentionFactorToken =
+ *     HMAC(HMAC(ESCTextExactToken, v), u)
+ * ESCTextSubstringDerivedFromDataTokenAndContentionFactorToken =
+ *     HMAC(HMAC(ESCTextSubstringToken, v), u)
+ * ESCTextSuffixDerivedFromDataTokenAndContentionFactorToken =
+ *     HMAC(HMAC(ESCTextSuffixToken, v), u)
+ * ESCTextPrefixDerivedFromDataTokenAndContentionFactorToken =
+ *     HMAC(HMAC(ESCTextPrefixToken, v), u)
+ *
+ * ServerTextExactDerivedFromDataToken = HMAC(ServerTextExactToken, v)
+ * ServerTextSubstringDerivedFromDataToken = HMAC(ServerTextSubstringToken, v)
+ * ServerTextSuffixDerivedFromDataToken = HMAC(ServerTextSuffixToken, v)
+ * ServerTextPrefixDerivedFromDataToken = HMAC(ServerTextPrefixToken, v)
+ *
  * ======================== End: FLE 2 Token Reference ========================
  */
 
@@ -89,8 +130,10 @@
     extern const _mongocrypt_buffer_t *BSON_CONCAT(Prefix, _get)(const T *t);                                          \
     /* Destructor */                                                                                                   \
     extern void BSON_CONCAT(Prefix, _destroy)(T * t);                                                                  \
-    /* Constructor for server to create tokens from raw buffer */                                                      \
-    extern T *BSON_CONCAT(Prefix, _new_from_buffer)(_mongocrypt_buffer_t * buf);                                       \
+    /* Constructor for server to shallow copy tokens from raw buffer */                                                \
+    extern T *BSON_CONCAT(Prefix, _new_from_buffer)(const _mongocrypt_buffer_t *buf);                                  \
+    /* Constructor for server to deep copy tokens from raw buffer */                                                   \
+    extern T *BSON_CONCAT(Prefix, _new_from_buffer_copy)(const _mongocrypt_buffer_t *buf);                             \
     /* Constructor. Parameter list given as variadic args */                                                           \
     extern T *BSON_CONCAT(Prefix, _new)(_mongocrypt_crypto_t * crypto, __VA_ARGS__, mongocrypt_status_t * status)
 
@@ -133,6 +176,91 @@ DECL_TOKEN_TYPE(mc_ServerCountAndContentionFactorEncryptionToken,
 DECL_TOKEN_TYPE(mc_ServerZerosEncryptionToken, const mc_ServerDerivedFromDataToken_t *serverDerivedFromDataToken);
 
 DECL_TOKEN_TYPE(mc_AnchorPaddingTokenRoot, const mc_ESCToken_t *ESCToken);
+DECL_TOKEN_TYPE(mc_AnchorPaddingKeyToken, const mc_AnchorPaddingTokenRoot_t *anchorPaddingToken);
+DECL_TOKEN_TYPE(mc_AnchorPaddingValueToken, const mc_AnchorPaddingTokenRoot_t *anchorPaddingToken);
+
+DECL_TOKEN_TYPE(mc_EDCTextExactToken, const mc_EDCToken_t *edcToken);
+DECL_TOKEN_TYPE(mc_EDCTextSubstringToken, const mc_EDCToken_t *edcToken);
+DECL_TOKEN_TYPE(mc_EDCTextSuffixToken, const mc_EDCToken_t *edcToken);
+DECL_TOKEN_TYPE(mc_EDCTextPrefixToken, const mc_EDCToken_t *edcToken);
+
+DECL_TOKEN_TYPE(mc_ESCTextExactToken, const mc_ESCToken_t *escToken);
+DECL_TOKEN_TYPE(mc_ESCTextSubstringToken, const mc_ESCToken_t *escToken);
+DECL_TOKEN_TYPE(mc_ESCTextSuffixToken, const mc_ESCToken_t *escToken);
+DECL_TOKEN_TYPE(mc_ESCTextPrefixToken, const mc_ESCToken_t *escToken);
+
+DECL_TOKEN_TYPE(mc_ServerTextExactToken, const mc_ServerTokenDerivationLevel1Token_t *serverTokenDerivationLevel1Token);
+DECL_TOKEN_TYPE(mc_ServerTextSubstringToken,
+                const mc_ServerTokenDerivationLevel1Token_t *serverTokenDerivationLevel1Token);
+DECL_TOKEN_TYPE(mc_ServerTextSuffixToken,
+                const mc_ServerTokenDerivationLevel1Token_t *serverTokenDerivationLevel1Token);
+DECL_TOKEN_TYPE(mc_ServerTextPrefixToken,
+                const mc_ServerTokenDerivationLevel1Token_t *serverTokenDerivationLevel1Token);
+
+DECL_TOKEN_TYPE(mc_EDCTextExactDerivedFromDataToken,
+                const mc_EDCTextExactToken_t *edcTextExactToken,
+                const _mongocrypt_buffer_t *v);
+DECL_TOKEN_TYPE(mc_EDCTextSubstringDerivedFromDataToken,
+                const mc_EDCTextSubstringToken_t *edcTextSubstringToken,
+                const _mongocrypt_buffer_t *v);
+DECL_TOKEN_TYPE(mc_EDCTextSuffixDerivedFromDataToken,
+                const mc_EDCTextSuffixToken_t *edcTextSuffixToken,
+                const _mongocrypt_buffer_t *v);
+DECL_TOKEN_TYPE(mc_EDCTextPrefixDerivedFromDataToken,
+                const mc_EDCTextPrefixToken_t *edcTextPrefixToken,
+                const _mongocrypt_buffer_t *v);
+
+DECL_TOKEN_TYPE(mc_EDCTextExactDerivedFromDataTokenAndContentionFactorToken,
+                const mc_EDCTextExactDerivedFromDataToken_t *edcTextExactToken,
+                uint64_t u);
+DECL_TOKEN_TYPE(mc_EDCTextSubstringDerivedFromDataTokenAndContentionFactorToken,
+                const mc_EDCTextSubstringDerivedFromDataToken_t *edcTextSubstringToken,
+                uint64_t u);
+DECL_TOKEN_TYPE(mc_EDCTextSuffixDerivedFromDataTokenAndContentionFactorToken,
+                const mc_EDCTextSuffixDerivedFromDataToken_t *edcTextSuffixToken,
+                uint64_t u);
+DECL_TOKEN_TYPE(mc_EDCTextPrefixDerivedFromDataTokenAndContentionFactorToken,
+                const mc_EDCTextPrefixDerivedFromDataToken_t *edcTextPrefixToken,
+                uint64_t u);
+
+DECL_TOKEN_TYPE(mc_ESCTextExactDerivedFromDataToken,
+                const mc_ESCTextExactToken_t *escTextExactToken,
+                const _mongocrypt_buffer_t *v);
+DECL_TOKEN_TYPE(mc_ESCTextSubstringDerivedFromDataToken,
+                const mc_ESCTextSubstringToken_t *escTextSubstringToken,
+                const _mongocrypt_buffer_t *v);
+DECL_TOKEN_TYPE(mc_ESCTextSuffixDerivedFromDataToken,
+                const mc_ESCTextSuffixToken_t *escTextSuffixToken,
+                const _mongocrypt_buffer_t *v);
+DECL_TOKEN_TYPE(mc_ESCTextPrefixDerivedFromDataToken,
+                const mc_ESCTextPrefixToken_t *escTextPrefixToken,
+                const _mongocrypt_buffer_t *v);
+
+DECL_TOKEN_TYPE(mc_ESCTextExactDerivedFromDataTokenAndContentionFactorToken,
+                const mc_ESCTextExactDerivedFromDataToken_t *escTextExactToken,
+                uint64_t u);
+DECL_TOKEN_TYPE(mc_ESCTextSubstringDerivedFromDataTokenAndContentionFactorToken,
+                const mc_ESCTextSubstringDerivedFromDataToken_t *escTextSubstringToken,
+                uint64_t u);
+DECL_TOKEN_TYPE(mc_ESCTextSuffixDerivedFromDataTokenAndContentionFactorToken,
+                const mc_ESCTextSuffixDerivedFromDataToken_t *escTextSuffixToken,
+                uint64_t u);
+DECL_TOKEN_TYPE(mc_ESCTextPrefixDerivedFromDataTokenAndContentionFactorToken,
+                const mc_ESCTextPrefixDerivedFromDataToken_t *escTextPrefixToken,
+                uint64_t u);
+
+DECL_TOKEN_TYPE(mc_ServerTextExactDerivedFromDataToken,
+                const mc_ServerTextExactToken_t *serverTextExactToken,
+                const _mongocrypt_buffer_t *v);
+DECL_TOKEN_TYPE(mc_ServerTextSubstringDerivedFromDataToken,
+                const mc_ServerTextSubstringToken_t *serverTextSubstringToken,
+                const _mongocrypt_buffer_t *v);
+DECL_TOKEN_TYPE(mc_ServerTextSuffixDerivedFromDataToken,
+                const mc_ServerTextSuffixToken_t *serverTextSuffixToken,
+                const _mongocrypt_buffer_t *v);
+DECL_TOKEN_TYPE(mc_ServerTextPrefixDerivedFromDataToken,
+                const mc_ServerTextPrefixToken_t *serverTextPrefixToken,
+                const _mongocrypt_buffer_t *v);
 
 #undef DECL_TOKEN_TYPE
 #undef DECL_TOKEN_TYPE_1

data/ext/libmongocrypt/libmongocrypt/src/mc-tokens.c

@@ -38,13 +38,21 @@
         _mongocrypt_buffer_cleanup(&self->data);                                                                       \
         bson_free(self);                                                                                               \
     }                                                                                                                  \
-    /* Constructor. From raw buffer */                                                                                 \
-    T *BSON_CONCAT(Prefix, _new_from_buffer)(_mongocrypt_buffer_t * buf) {                                             \
+    /* Constructor. Shallow copy from raw buffer */                                                                    \
+    T *BSON_CONCAT(Prefix, _new_from_buffer)(const _mongocrypt_buffer_t *buf) {                                        \
         BSON_ASSERT(buf->len == MONGOCRYPT_HMAC_SHA256_LEN);                                                           \
         T *t = bson_malloc(sizeof(T));                                                                                 \
         _mongocrypt_buffer_set_to(buf, &t->data);                                                                      \
         return t;                                                                                                      \
     }                                                                                                                  \
+    /* Constructor. Deep copy from raw buffer */                                                                       \
+    T *BSON_CONCAT(Prefix, _new_from_buffer_copy)(const _mongocrypt_buffer_t *buf) {                                   \
+        BSON_ASSERT(buf->len == MONGOCRYPT_HMAC_SHA256_LEN);                                                           \
+        T *t = bson_malloc(sizeof(T));                                                                                 \
+        _mongocrypt_buffer_init(&t->data);                                                                             \
+        _mongocrypt_buffer_copy_to(buf, &t->data);                                                                     \
+        return t;                                                                                                      \
+    }                                                                                                                  \
     /* Constructor. Parameter list given as variadic args. */                                                          \
     T *BSON_CONCAT(Prefix, _new)(_mongocrypt_crypto_t * crypto, __VA_ARGS__, mongocrypt_status_t * status)
 
@@ -160,7 +168,7 @@ IMPL_TOKEN_NEW_CONST(mc_ServerZerosEncryptionToken, mc_ServerDerivedFromDataToke
 
 // d = 17 bytes of 0, AnchorPaddingTokenRoot = HMAC(ESCToken, d)
 #define ANCHOR_PADDING_TOKEN_D_LENGTH 17
-const uint8_t mc_AnchorPaddingTokenDValue[ANCHOR_PADDING_TOKEN_D_LENGTH] =
+static const uint8_t mc_AnchorPaddingTokenDValue[ANCHOR_PADDING_TOKEN_D_LENGTH] =
     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
 DEF_TOKEN_TYPE(mc_AnchorPaddingTokenRoot, const mc_ESCToken_t *ESCToken) {
@@ -177,3 +185,151 @@ DEF_TOKEN_TYPE(mc_AnchorPaddingTokenRoot, const mc_ESCToken_t *ESCToken) {
 }
 
 #undef ANCHOR_PADDING_TOKEN_D_LENGTH
+
+DEF_TOKEN_TYPE(mc_AnchorPaddingKeyToken, const mc_AnchorPaddingTokenRoot_t *anchorPaddingToken)
+IMPL_TOKEN_NEW_CONST(mc_AnchorPaddingKeyToken, mc_AnchorPaddingTokenRoot_get(anchorPaddingToken), 1)
+
+DEF_TOKEN_TYPE(mc_AnchorPaddingValueToken, const mc_AnchorPaddingTokenRoot_t *anchorPaddingToken)
+IMPL_TOKEN_NEW_CONST(mc_AnchorPaddingValueToken, mc_AnchorPaddingTokenRoot_get(anchorPaddingToken), 2)
+
+#define TEXT_EXACT_ID 1
+#define TEXT_SUBSTRING_ID 2
+#define TEXT_SUFFIX_ID 3
+#define TEXT_PREFIX_ID 4
+
+DEF_TOKEN_TYPE(mc_EDCTextExactToken, const mc_EDCToken_t *edcToken)
+IMPL_TOKEN_NEW_CONST(mc_EDCTextExactToken, mc_EDCToken_get(edcToken), TEXT_EXACT_ID)
+DEF_TOKEN_TYPE(mc_EDCTextSubstringToken, const mc_EDCToken_t *edcToken)
+IMPL_TOKEN_NEW_CONST(mc_EDCTextSubstringToken, mc_EDCToken_get(edcToken), TEXT_SUBSTRING_ID)
+DEF_TOKEN_TYPE(mc_EDCTextSuffixToken, const mc_EDCToken_t *edcToken)
+IMPL_TOKEN_NEW_CONST(mc_EDCTextSuffixToken, mc_EDCToken_get(edcToken), TEXT_SUFFIX_ID)
+DEF_TOKEN_TYPE(mc_EDCTextPrefixToken, const mc_EDCToken_t *edcToken)
+IMPL_TOKEN_NEW_CONST(mc_EDCTextPrefixToken, mc_EDCToken_get(edcToken), TEXT_PREFIX_ID)
+
+DEF_TOKEN_TYPE(mc_ESCTextExactToken, const mc_ESCToken_t *escToken)
+IMPL_TOKEN_NEW_CONST(mc_ESCTextExactToken, mc_ESCToken_get(escToken), TEXT_EXACT_ID)
+DEF_TOKEN_TYPE(mc_ESCTextSubstringToken, const mc_ESCToken_t *escToken)
+IMPL_TOKEN_NEW_CONST(mc_ESCTextSubstringToken, mc_ESCToken_get(escToken), TEXT_SUBSTRING_ID)
+DEF_TOKEN_TYPE(mc_ESCTextSuffixToken, const mc_ESCToken_t *escToken)
+IMPL_TOKEN_NEW_CONST(mc_ESCTextSuffixToken, mc_ESCToken_get(escToken), TEXT_SUFFIX_ID)
+DEF_TOKEN_TYPE(mc_ESCTextPrefixToken, const mc_ESCToken_t *escToken)
+IMPL_TOKEN_NEW_CONST(mc_ESCTextPrefixToken, mc_ESCToken_get(escToken), TEXT_PREFIX_ID)
+
+DEF_TOKEN_TYPE(mc_ServerTextExactToken, const mc_ServerTokenDerivationLevel1Token_t *serverTokenDerivationLevel1Token)
+IMPL_TOKEN_NEW_CONST(mc_ServerTextExactToken,
+                     mc_ServerTokenDerivationLevel1Token_get(serverTokenDerivationLevel1Token),
+                     TEXT_EXACT_ID)
+DEF_TOKEN_TYPE(mc_ServerTextSubstringToken,
+               const mc_ServerTokenDerivationLevel1Token_t *serverTokenDerivationLevel1Token)
+IMPL_TOKEN_NEW_CONST(mc_ServerTextSubstringToken,
+                     mc_ServerTokenDerivationLevel1Token_get(serverTokenDerivationLevel1Token),
+                     TEXT_SUBSTRING_ID)
+DEF_TOKEN_TYPE(mc_ServerTextSuffixToken, const mc_ServerTokenDerivationLevel1Token_t *serverTokenDerivationLevel1Token)
+IMPL_TOKEN_NEW_CONST(mc_ServerTextSuffixToken,
+                     mc_ServerTokenDerivationLevel1Token_get(serverTokenDerivationLevel1Token),
+                     TEXT_SUFFIX_ID)
+DEF_TOKEN_TYPE(mc_ServerTextPrefixToken, const mc_ServerTokenDerivationLevel1Token_t *serverTokenDerivationLevel1Token)
+IMPL_TOKEN_NEW_CONST(mc_ServerTextPrefixToken,
+                     mc_ServerTokenDerivationLevel1Token_get(serverTokenDerivationLevel1Token),
+                     TEXT_PREFIX_ID)
+
+DEF_TOKEN_TYPE(mc_EDCTextExactDerivedFromDataToken,
+               const mc_EDCTextExactToken_t *edcTextExactToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_EDCTextExactDerivedFromDataToken, mc_EDCTextExactToken_get(edcTextExactToken), v)
+DEF_TOKEN_TYPE(mc_EDCTextSubstringDerivedFromDataToken,
+               const mc_EDCTextSubstringToken_t *edcTextSubstringToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_EDCTextSubstringDerivedFromDataToken, mc_EDCTextSubstringToken_get(edcTextSubstringToken), v)
+DEF_TOKEN_TYPE(mc_EDCTextSuffixDerivedFromDataToken,
+               const mc_EDCTextSuffixToken_t *edcTextSuffixToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_EDCTextSuffixDerivedFromDataToken, mc_EDCTextSuffixToken_get(edcTextSuffixToken), v)
+DEF_TOKEN_TYPE(mc_EDCTextPrefixDerivedFromDataToken,
+               const mc_EDCTextPrefixToken_t *edcTextPrefixToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_EDCTextPrefixDerivedFromDataToken, mc_EDCTextPrefixToken_get(edcTextPrefixToken), v)
+
+DEF_TOKEN_TYPE(mc_EDCTextExactDerivedFromDataTokenAndContentionFactorToken,
+               const mc_EDCTextExactDerivedFromDataToken_t *edcTextExactToken,
+               uint64_t u)
+IMPL_TOKEN_NEW_CONST(mc_EDCTextExactDerivedFromDataTokenAndContentionFactorToken,
+                     mc_EDCTextExactDerivedFromDataToken_get(edcTextExactToken),
+                     u)
+DEF_TOKEN_TYPE(mc_EDCTextSubstringDerivedFromDataTokenAndContentionFactorToken,
+               const mc_EDCTextSubstringDerivedFromDataToken_t *edcTextSubstringToken,
+               uint64_t u)
+IMPL_TOKEN_NEW_CONST(mc_EDCTextSubstringDerivedFromDataTokenAndContentionFactorToken,
+                     mc_EDCTextSubstringDerivedFromDataToken_get(edcTextSubstringToken),
+                     u)
+DEF_TOKEN_TYPE(mc_EDCTextSuffixDerivedFromDataTokenAndContentionFactorToken,
+               const mc_EDCTextSuffixDerivedFromDataToken_t *edcTextSuffixToken,
+               uint64_t u)
+IMPL_TOKEN_NEW_CONST(mc_EDCTextSuffixDerivedFromDataTokenAndContentionFactorToken,
+                     mc_EDCTextSuffixDerivedFromDataToken_get(edcTextSuffixToken),
+                     u)
+DEF_TOKEN_TYPE(mc_EDCTextPrefixDerivedFromDataTokenAndContentionFactorToken,
+               const mc_EDCTextPrefixDerivedFromDataToken_t *edcTextPrefixToken,
+               uint64_t u)
+IMPL_TOKEN_NEW_CONST(mc_EDCTextPrefixDerivedFromDataTokenAndContentionFactorToken,
+                     mc_EDCTextPrefixDerivedFromDataToken_get(edcTextPrefixToken),
+                     u)
+
+DEF_TOKEN_TYPE(mc_ESCTextExactDerivedFromDataToken,
+               const mc_ESCTextExactToken_t *escTextExactToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_ESCTextExactDerivedFromDataToken, mc_ESCTextExactToken_get(escTextExactToken), v)
+DEF_TOKEN_TYPE(mc_ESCTextSubstringDerivedFromDataToken,
+               const mc_ESCTextSubstringToken_t *escTextSubstringToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_ESCTextSubstringDerivedFromDataToken, mc_ESCTextSubstringToken_get(escTextSubstringToken), v)
+DEF_TOKEN_TYPE(mc_ESCTextSuffixDerivedFromDataToken,
+               const mc_ESCTextSuffixToken_t *escTextSuffixToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_ESCTextSuffixDerivedFromDataToken, mc_ESCTextSuffixToken_get(escTextSuffixToken), v)
+DEF_TOKEN_TYPE(mc_ESCTextPrefixDerivedFromDataToken,
+               const mc_ESCTextPrefixToken_t *escTextPrefixToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_ESCTextPrefixDerivedFromDataToken, mc_ESCTextPrefixToken_get(escTextPrefixToken), v)
+
+DEF_TOKEN_TYPE(mc_ESCTextExactDerivedFromDataTokenAndContentionFactorToken,
+               const mc_ESCTextExactDerivedFromDataToken_t *escTextExactToken,
+               uint64_t u)
+IMPL_TOKEN_NEW_CONST(mc_ESCTextExactDerivedFromDataTokenAndContentionFactorToken,
+                     mc_ESCTextExactDerivedFromDataToken_get(escTextExactToken),
+                     u)
+DEF_TOKEN_TYPE(mc_ESCTextSubstringDerivedFromDataTokenAndContentionFactorToken,
+               const mc_ESCTextSubstringDerivedFromDataToken_t *escTextSubstringToken,
+               uint64_t u)
+IMPL_TOKEN_NEW_CONST(mc_ESCTextSubstringDerivedFromDataTokenAndContentionFactorToken,
+                     mc_ESCTextSubstringDerivedFromDataToken_get(escTextSubstringToken),
+                     u)
+DEF_TOKEN_TYPE(mc_ESCTextSuffixDerivedFromDataTokenAndContentionFactorToken,
+               const mc_ESCTextSuffixDerivedFromDataToken_t *escTextSuffixToken,
+               uint64_t u)
+IMPL_TOKEN_NEW_CONST(mc_ESCTextSuffixDerivedFromDataTokenAndContentionFactorToken,
+                     mc_ESCTextSuffixDerivedFromDataToken_get(escTextSuffixToken),
+                     u)
+DEF_TOKEN_TYPE(mc_ESCTextPrefixDerivedFromDataTokenAndContentionFactorToken,
+               const mc_ESCTextPrefixDerivedFromDataToken_t *escTextPrefixToken,
+               uint64_t u)
+IMPL_TOKEN_NEW_CONST(mc_ESCTextPrefixDerivedFromDataTokenAndContentionFactorToken,
+                     mc_ESCTextPrefixDerivedFromDataToken_get(escTextPrefixToken),
+                     u)
+
+DEF_TOKEN_TYPE(mc_ServerTextExactDerivedFromDataToken,
+               const mc_ServerTextExactToken_t *serverTextExactToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_ServerTextExactDerivedFromDataToken, mc_ServerTextExactToken_get(serverTextExactToken), v)
+DEF_TOKEN_TYPE(mc_ServerTextSubstringDerivedFromDataToken,
+               const mc_ServerTextSubstringToken_t *serverTextSubstringToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_ServerTextSubstringDerivedFromDataToken, mc_ServerTextSubstringToken_get(serverTextSubstringToken), v)
+DEF_TOKEN_TYPE(mc_ServerTextSuffixDerivedFromDataToken,
+               const mc_ServerTextSuffixToken_t *serverTextSuffixToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_ServerTextSuffixDerivedFromDataToken, mc_ServerTextSuffixToken_get(serverTextSuffixToken), v)
+DEF_TOKEN_TYPE(mc_ServerTextPrefixDerivedFromDataToken,
+               const mc_ServerTextPrefixToken_t *serverTextPrefixToken,
+               const _mongocrypt_buffer_t *v)
+IMPL_TOKEN_NEW(mc_ServerTextPrefixDerivedFromDataToken, mc_ServerTextPrefixToken_get(serverTextPrefixToken), v)

data/ext/libmongocrypt/libmongocrypt/src/mc-writer-private.h

@@ -25,7 +25,7 @@
  * A non-owning forward-only cursor api to write to a buffer.
  *
  * Tracks length of buffer and current position of buffer. parser_name is
- * typically __FUNCTION__ to provide useful error messages automatically.
+ * typically __func__ to provide useful error messages automatically.
  *
  * All numbers are written as little endian.
  */