duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/third_party/snappy/snappy.h

@@ -39,6 +39,227 @@
 #ifndef THIRD_PARTY_SNAPPY_SNAPPY_H__
 #define THIRD_PARTY_SNAPPY_SNAPPY_H__
 
+#include "snappy_version.hpp"
+
+#if SNAPPY_NEW_VERSION
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <string>
+
+#include "snappy-stubs-public.h"
+
+namespace duckdb_snappy {
+  class Source;
+  class Sink;
+
+  struct CompressionOptions {
+    // Compression level.
+    // Level 1 is the fastest
+    // Level 2 is a little slower but provides better compression. Level 2 is
+    // **EXPERIMENTAL** for the time being. It might happen that we decide to
+    // fall back to level 1 in the future.
+    // Levels 3+ are currently not supported. We plan to support levels up to
+    // 9 in the future.
+    // If you played with other compression algorithms, level 1 is equivalent to
+    // fast mode (level 1) of LZ4, level 2 is equivalent to LZ4's level 2 mode
+    // and compresses somewhere around zstd:-3 and zstd:-2 but generally with
+    // faster decompression speeds than snappy:1 and zstd:-3.
+    int level = DefaultCompressionLevel();
+
+    constexpr CompressionOptions() = default;
+    constexpr CompressionOptions(int compression_level)
+        : level(compression_level) {}
+    static constexpr int MinCompressionLevel() { return 1; }
+    static constexpr int MaxCompressionLevel() { return 2; }
+    static constexpr int DefaultCompressionLevel() { return 1; }
+  };
+
+  // ------------------------------------------------------------------------
+  // Generic compression/decompression routines.
+  // ------------------------------------------------------------------------
+
+  // Compress the bytes read from "*reader" and append to "*writer". Return the
+  // number of bytes written.
+  // First version is to preserve ABI.
+  size_t Compress(Source* reader, Sink* writer);
+  size_t Compress(Source* reader, Sink* writer,
+                  CompressionOptions options);
+
+  // Find the uncompressed length of the given stream, as given by the header.
+  // Note that the true length could deviate from this; the stream could e.g.
+  // be truncated.
+  //
+  // Also note that this leaves "*source" in a state that is unsuitable for
+  // further operations, such as RawUncompress(). You will need to rewind
+  // or recreate the source yourself before attempting any further calls.
+  bool GetUncompressedLength(Source* source, uint32_t* result);
+
+  // ------------------------------------------------------------------------
+  // Higher-level string based routines (should be sufficient for most users)
+  // ------------------------------------------------------------------------
+
+  // Sets "*compressed" to the compressed version of "input[0..input_length-1]".
+  // Original contents of *compressed are lost.
+  //
+  // REQUIRES: "input[]" is not an alias of "*compressed".
+  // First version is to preserve ABI.
+  size_t Compress(const char* input, size_t input_length,
+                  std::string* compressed);
+  size_t Compress(const char* input, size_t input_length,
+                  std::string* compressed, CompressionOptions options);
+
+  // Same as `Compress` above but taking an `iovec` array as input. Note that
+  // this function preprocesses the inputs to compute the sum of
+  // `iov[0..iov_cnt-1].iov_len` before reading. To avoid this, use
+  // `RawCompressFromIOVec` below.
+  // First version is to preserve ABI.
+  size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt,
+                           std::string* compressed);
+  size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt,
+                           std::string* compressed,
+                           CompressionOptions options);
+
+  // Decompresses "compressed[0..compressed_length-1]" to "*uncompressed".
+  // Original contents of "*uncompressed" are lost.
+  //
+  // REQUIRES: "compressed[]" is not an alias of "*uncompressed".
+  //
+  // returns false if the message is corrupted and could not be decompressed
+  bool Uncompress(const char* compressed, size_t compressed_length,
+                  std::string* uncompressed);
+
+  // Decompresses "compressed" to "*uncompressed".
+  //
+  // returns false if the message is corrupted and could not be decompressed
+  bool Uncompress(Source* compressed, Sink* uncompressed);
+
+  // This routine uncompresses as much of the "compressed" as possible
+  // into sink.  It returns the number of valid bytes added to sink
+  // (extra invalid bytes may have been added due to errors; the caller
+  // should ignore those). The emitted data typically has length
+  // GetUncompressedLength(), but may be shorter if an error is
+  // encountered.
+  size_t UncompressAsMuchAsPossible(Source* compressed, Sink* uncompressed);
+
+  // ------------------------------------------------------------------------
+  // Lower-level character array based routines.  May be useful for
+  // efficiency reasons in certain circumstances.
+  // ------------------------------------------------------------------------
+
+  // REQUIRES: "compressed" must point to an area of memory that is at
+  // least "MaxCompressedLength(input_length)" bytes in length.
+  //
+  // Takes the data stored in "input[0..input_length]" and stores
+  // it in the array pointed to by "compressed".
+  //
+  // "*compressed_length" is set to the length of the compressed output.
+  //
+  // Example:
+  //    char* output = new char[snappy::MaxCompressedLength(input_length)];
+  //    size_t output_length;
+  //    RawCompress(input, input_length, output, &output_length);
+  //    ... Process(output, output_length) ...
+  //    delete [] output;
+  void RawCompress(const char* input, size_t input_length, char* compressed,
+                   size_t* compressed_length);
+  void RawCompress(const char* input, size_t input_length, char* compressed,
+                   size_t* compressed_length, CompressionOptions options);
+
+  // Same as `RawCompress` above but taking an `iovec` array as input. Note that
+  // `uncompressed_length` is the total number of bytes to be read from the
+  // elements of `iov` (_not_ the number of elements in `iov`).
+  void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length,
+                            char* compressed, size_t* compressed_length);
+  void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length,
+                            char* compressed, size_t* compressed_length,
+                            CompressionOptions options);
+
+  // Given data in "compressed[0..compressed_length-1]" generated by
+  // calling the Snappy::Compress routine, this routine
+  // stores the uncompressed data to
+  //    uncompressed[0..GetUncompressedLength(compressed)-1]
+  // returns false if the message is corrupted and could not be decrypted
+  bool RawUncompress(const char* compressed, size_t compressed_length,
+                     char* uncompressed);
+
+  // Given data from the byte source 'compressed' generated by calling
+  // the Snappy::Compress routine, this routine stores the uncompressed
+  // data to
+  //    uncompressed[0..GetUncompressedLength(compressed,compressed_length)-1]
+  // returns false if the message is corrupted and could not be decrypted
+  bool RawUncompress(Source* compressed, char* uncompressed);
+
+  // Given data in "compressed[0..compressed_length-1]" generated by
+  // calling the Snappy::Compress routine, this routine
+  // stores the uncompressed data to the iovec "iov". The number of physical
+  // buffers in "iov" is given by iov_cnt and their cumulative size
+  // must be at least GetUncompressedLength(compressed). The individual buffers
+  // in "iov" must not overlap with each other.
+  //
+  // returns false if the message is corrupted and could not be decrypted
+  bool RawUncompressToIOVec(const char* compressed, size_t compressed_length,
+                            const struct iovec* iov, size_t iov_cnt);
+
+  // Given data from the byte source 'compressed' generated by calling
+  // the Snappy::Compress routine, this routine stores the uncompressed
+  // data to the iovec "iov". The number of physical
+  // buffers in "iov" is given by iov_cnt and their cumulative size
+  // must be at least GetUncompressedLength(compressed). The individual buffers
+  // in "iov" must not overlap with each other.
+  //
+  // returns false if the message is corrupted and could not be decrypted
+  bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov,
+                            size_t iov_cnt);
+
+  // Returns the maximal size of the compressed representation of
+  // input data that is "source_bytes" bytes in length;
+  size_t MaxCompressedLength(size_t source_bytes);
+
+  // REQUIRES: "compressed[]" was produced by RawCompress() or Compress()
+  // Returns true and stores the length of the uncompressed data in
+  // *result normally.  Returns false on parsing error.
+  // This operation takes O(1) time.
+  bool GetUncompressedLength(const char* compressed, size_t compressed_length,
+                             size_t* result);
+
+  // Returns true iff the contents of "compressed[]" can be uncompressed
+  // successfully.  Does not return the uncompressed data.  Takes
+  // time proportional to compressed_length, but is usually at least
+  // a factor of four faster than actual decompression.
+  bool IsValidCompressedBuffer(const char* compressed,
+                               size_t compressed_length);
+
+  // Returns true iff the contents of "compressed" can be uncompressed
+  // successfully.  Does not return the uncompressed data.  Takes
+  // time proportional to *compressed length, but is usually at least
+  // a factor of four faster than actual decompression.
+  // On success, consumes all of *compressed.  On failure, consumes an
+  // unspecified prefix of *compressed.
+  bool IsValidCompressed(Source* compressed);
+
+  // The size of a compression block. Note that many parts of the compression
+  // code assumes that kBlockSize <= 65536; in particular, the hash table
+  // can only store 16-bit offsets, and EmitCopy() also assumes the offset
+  // is 65535 bytes or less. Note also that if you change this, it will
+  // affect the framing format (see framing_format.txt).
+  //
+  // Note that there might be older data around that is compressed with larger
+  // block sizes, so the decompression code should not rely on the
+  // non-existence of long backreferences.
+  static constexpr int kBlockLog = 16;
+  static constexpr size_t kBlockSize = 1 << kBlockLog;
+
+  static constexpr int kMinHashTableBits = 8;
+  static constexpr size_t kMinHashTableSize = 1 << kMinHashTableBits;
+
+  static constexpr int kMaxHashTableBits = 15;
+  static constexpr size_t kMaxHashTableSize = 1 << kMaxHashTableBits;
+}  // end namespace duckdb_snappy
+
+#else // #if SNAPPY_NEW_VERSION
+
 #include <cstddef>
 #include <string>
 
@@ -186,4 +407,6 @@ namespace duckdb_snappy {
 
 }  // end namespace duckdb_snappy
 
+#endif  // #if SNAPPY_NEW_VERSION # else
+
 #endif  // THIRD_PARTY_SNAPPY_SNAPPY_H__

package/src/duckdb/third_party/snappy/snappy_version.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+//! The new version of snappy is much faster when compiled with clang, but slower when compiled with GCC
+//! For DuckDB, we default to the old version if the compiler is not clang
+#ifndef SNAPPY_NEW_VERSION
+#ifdef __clang__
+#define SNAPPY_NEW_VERSION true
+#else
+#define SNAPPY_NEW_VERSION false
+#endif
+#endif

package/src/duckdb/third_party/utf8proc/include/utf8proc.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2019 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
+ * Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
  * Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -36,16 +36,16 @@
  *
  * The features of utf8proc include:
  *
- * - Transformation of strings (@ref utf8proc_map) to:
+ * - Transformation of strings (utf8proc_map()) to:
  *    - decompose (@ref UTF8PROC_DECOMPOSE) or compose (@ref UTF8PROC_COMPOSE) Unicode combining characters (http://en.wikipedia.org/wiki/Combining_character)
  *    - canonicalize Unicode compatibility characters (@ref UTF8PROC_COMPAT)
  *    - strip "ignorable" (@ref UTF8PROC_IGNORE) characters, control characters (@ref UTF8PROC_STRIPCC), or combining characters such as accents (@ref UTF8PROC_STRIPMARK)
  *    - case-folding (@ref UTF8PROC_CASEFOLD)
- * - Unicode normalization: @ref utf8proc_NFD, @ref utf8proc_NFC, @ref utf8proc_NFKD, @ref utf8proc_NFKC
- * - Detecting grapheme boundaries (@ref utf8proc_grapheme_break and @ref UTF8PROC_CHARBOUND)
- * - Character-width computation: @ref utf8proc_charwidth
- * - Classification of characters by Unicode category: @ref utf8proc_category and @ref utf8proc_category_string
- * - Encode (@ref utf8proc_encode_char) and decode (@ref utf8proc_iterate) Unicode codepoints to/from UTF-8.
+ * - Unicode normalization: utf8proc_NFD(), utf8proc_NFC(), utf8proc_NFKD(), utf8proc_NFKC()
+ * - Detecting grapheme boundaries (utf8proc_grapheme_break() and @ref UTF8PROC_CHARBOUND)
+ * - Character-width computation: utf8proc_charwidth()
+ * - Classification of characters by Unicode category: utf8proc_category() and utf8proc_category_string()
+ * - Encode (utf8proc_encode_char()) and decode (utf8proc_iterate()) Unicode codepoints to/from UTF-8.
  */
 
 /** @file */
@@ -53,16 +53,13 @@
 #ifndef UTF8PROC_H
 #define UTF8PROC_H
 
-// DuckDB change:
-#define UTF8PROC_STATIC
-
 /** @name API version
  *
  * The utf8proc API version MAJOR.MINOR.PATCH, following
  * semantic-versioning rules (http://semver.org) based on API
  * compatibility.
  *
- * This is also returned at runtime by @ref utf8proc_version; however, the
+ * This is also returned at runtime by utf8proc_version(); however, the
  * runtime version may append a string like "-dev" to the version number
  * for prerelease versions.
  *
@@ -74,7 +71,7 @@
 /** The MAJOR version number (increased when backwards API compatibility is broken). */
 #define UTF8PROC_VERSION_MAJOR 2
 /** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */
-#define UTF8PROC_VERSION_MINOR 4
+#define UTF8PROC_VERSION_MINOR 9
 /** The PATCH version (increased for fixes that do not change the API). */
 #define UTF8PROC_VERSION_PATCH 0
 /** @} */
@@ -82,7 +79,7 @@
 #include <stdlib.h>
 
 #if defined(_MSC_VER) && _MSC_VER < 1800
-// MSVC prior to 2013 lacked stdbool.h and inttypes.h
+// MSVC prior to 2013 lacked stdbool.h and stdint.h
 typedef signed char utf8proc_int8_t;
 typedef unsigned char utf8proc_uint8_t;
 typedef short utf8proc_int16_t;
@@ -110,29 +107,7 @@ typedef bool utf8proc_bool;
 #else
 #  include <stddef.h>
 #  include <stdbool.h>
-#  include <inttypes.h>
-#endif
-#include <limits.h>
-
-#define UTF8PROC_DLLEXPORT
-// #ifdef UTF8PROC_STATIC
-// #  define UTF8PROC_DLLEXPORT
-// #else
-// #  ifdef _WIN32
-// #    ifdef UTF8PROC_EXPORTS
-// #      define UTF8PROC_DLLEXPORT __declspec(dllexport)
-// #    else
-// #      define UTF8PROC_DLLEXPORT __declspec(dllimport)
-// #    endif
-// #  elif __GNUC__ >= 4
-// #    define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default")))
-// #  else
-// #    define UTF8PROC_DLLEXPORT
-// #  endif
-// #endif
-
-namespace duckdb {
-
+#  include <stdint.h>
 typedef int8_t utf8proc_int8_t;
 typedef uint8_t utf8proc_uint8_t;
 typedef int16_t utf8proc_int16_t;
@@ -142,10 +117,12 @@ typedef uint32_t utf8proc_uint32_t;
 typedef size_t utf8proc_size_t;
 typedef ptrdiff_t utf8proc_ssize_t;
 typedef bool utf8proc_bool;
+#endif
+#include <limits.h>
 
-//#ifdef __cplusplus
-//extern "C" {
-//#endif
+#  define UTF8PROC_DLLEXPORT
+
+namespace duckdb {
 
 /**
  * Option flags used by several functions in the library.
@@ -268,7 +245,7 @@ typedef struct utf8proc_property_struct {
   /**
    * Can this codepoint be ignored?
    *
-   * Used by @ref utf8proc_decompose_char when @ref UTF8PROC_IGNORE is
+   * Used by utf8proc_decompose_char() when @ref UTF8PROC_IGNORE is
    * passed as an option.
    */
   unsigned ignorable:1;
@@ -280,7 +257,8 @@ typedef struct utf8proc_property_struct {
    * Boundclass.
    * @see utf8proc_boundclass_t.
    */
-  unsigned boundclass:8;
+  unsigned boundclass:6;
+  unsigned indic_conjunct_break:2;
 } utf8proc_property_t;
 
 /** Unicode categories. */
@@ -395,9 +373,17 @@ typedef enum {
   UTF8PROC_BOUNDCLASS_E_ZWG = 20, /* UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC + ZWJ */
 } utf8proc_boundclass_t;
 
+/** Indic_Conjunct_Break property. (TR44) */
+typedef enum {
+  UTF8PROC_INDIC_CONJUNCT_BREAK_NONE = 0,
+  UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER = 1,
+  UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT = 2,
+  UTF8PROC_INDIC_CONJUNCT_BREAK_EXTEND = 3,
+} utf8proc_indic_conjunct_break_t;
+
 /**
- * Function pointer type passed to @ref utf8proc_map_custom and
- * @ref utf8proc_decompose_custom, which is used to specify a user-defined
+ * Function pointer type passed to utf8proc_map_custom() and
+ * utf8proc_decompose_custom(), which is used to specify a user-defined
  * mapping of codepoints to be applied in conjunction with other mappings.
  */
 typedef utf8proc_int32_t (*utf8proc_custom_func)(utf8proc_int32_t codepoint, void *data);
@@ -406,7 +392,7 @@ typedef utf8proc_int32_t (*utf8proc_custom_func)(utf8proc_int32_t codepoint, voi
  * Array containing the byte lengths of a UTF-8 encoded codepoint based
  * on the first byte.
  */
-// UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256];
+UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256];
 
 /**
  * Returns the utf8proc API version as a string MAJOR.MINOR.PATCH
@@ -422,7 +408,7 @@ UTF8PROC_DLLEXPORT const char *utf8proc_unicode_version(void);
 
 /**
  * Returns an informative error string for the given utf8proc error code
- * (e.g. the error codes returned by @ref utf8proc_map).
+ * (e.g. the error codes returned by utf8proc_map()).
  */
 UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode);
 
@@ -488,12 +474,13 @@ UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int
  * - @ref UTF8PROC_STRIPNA   - remove unassigned codepoints
  * @param last_boundclass
  * Pointer to an integer variable containing
- * the previous codepoint's boundary class if the @ref UTF8PROC_CHARBOUND
- * option is used.  Otherwise, this parameter is ignored.
+ * the previous codepoint's (boundclass + indic_conjunct_break << 1) if the @ref UTF8PROC_CHARBOUND
+ * option is used.  If the string is being processed in order, this can be initialized to 0 for
+ * the beginning of the string, and is thereafter updated automatically.  Otherwise, this parameter is ignored.
  *
  * @return
  * In case of success, the number of codepoints written is returned; in case
- * of an error, a negative error code is returned (@ref utf8proc_errmsg).
+ * of an error, a negative error code is returned (utf8proc_errmsg()).
  * @par
  * If the number of written codepoints would be bigger than `bufsize`, the
  * required buffer size is returned, while the buffer will be overwritten with
@@ -505,17 +492,17 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(
 );
 
 /**
- * The same as @ref utf8proc_decompose_char, but acts on a whole UTF-8
+ * The same as utf8proc_decompose_char(), but acts on a whole UTF-8
  * string and orders the decomposed sequences correctly.
  *
  * If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing
- * will be stopped, when a NULL byte is encounted, otherwise `strlen`
+ * will be stopped, when a NULL byte is encountered, otherwise `strlen`
  * bytes are processed.  The result (in the form of 32-bit unicode
  * codepoints) is written into the buffer being pointed to by
  * `buffer` (which must contain at least `bufsize` entries).  In case of
  * success, the number of codepoints written is returned; in case of an
- * error, a negative error code is returned (@ref utf8proc_errmsg).
- * See @ref utf8proc_decompose_custom to supply additional transformations.
+ * error, a negative error code is returned (utf8proc_errmsg()).
+ * See utf8proc_decompose_custom() to supply additional transformations.
  *
  * If the number of written codepoints would be bigger than `bufsize`, the
  * required buffer size is returned, while the buffer will be overwritten with
@@ -527,10 +514,10 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
 );
 
 /**
- * The same as @ref utf8proc_decompose, but also takes a `custom_func` mapping function
+ * The same as utf8proc_decompose(), but also takes a `custom_func` mapping function
  * that is called on each codepoint in `str` before any other transformations
  * (along with a `custom_data` pointer that is passed through to `custom_func`).
- * The `custom_func` argument is ignored if it is `NULL`.  See also @ref utf8proc_map_custom.
+ * The `custom_func` argument is ignored if it is `NULL`.  See also utf8proc_map_custom().
  */
 UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
   const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
@@ -556,7 +543,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
  *
  * @return
  * In case of success, the length (in codepoints) of the normalized UTF-32 string is
- * returned; otherwise, a negative error code is returned (@ref utf8proc_errmsg).
+ * returned; otherwise, a negative error code is returned (utf8proc_errmsg()).
  *
  * @warning The entries of the array pointed to by `str` have to be in the
  *          range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
@@ -584,7 +571,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *b
  * @return
  * In case of success, the length (in bytes) of the resulting nul-terminated
  * UTF-8 string is returned; otherwise, a negative error code is returned
- * (@ref utf8proc_errmsg).
+ * (utf8proc_errmsg()).
  *
  * @warning The amount of free space pointed to by `buffer` must
  *          exceed the amount of the input data by one byte, and the
@@ -614,43 +601,12 @@ UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
     utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state);
 
 /**
- * Same as @ref utf8proc_grapheme_break_stateful, except without support for the
+ * Same as utf8proc_grapheme_break_stateful(), except without support for the
  * Unicode 9 additions to the algorithm. Supported for legacy reasons.
  */
 UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
     utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2);
 
-//! Returns the current UTF8 codepoint in a UTF8 string. Assumes the string is valid UTF8.
-UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_codepoint(const char *u_input, int &sz);
-UTF8PROC_DLLEXPORT utf8proc_bool grapheme_break_extended(int lbc, int tbc, utf8proc_int32_t *state);
-UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_codepoint(const char *u_input, int &sz);
-UTF8PROC_DLLEXPORT bool utf8proc_codepoint_to_utf8(int cp, int &sz, char *c);
-UTF8PROC_DLLEXPORT int utf8proc_codepoint_length(int cp);
-UTF8PROC_DLLEXPORT size_t utf8proc_next_grapheme(const char *s, size_t len, size_t cpos);
-UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_remove_accents(const utf8proc_uint8_t *str, utf8proc_ssize_t len);
-template<class T>
-void utf8proc_grapheme_callback(const char *s, size_t len, T &&fun) {
-	int sz;
-	int boundclass = UTF8PROC_BOUNDCLASS_START;
-	int initial = utf8proc_get_property(utf8proc_codepoint(s, sz))->boundclass;
-	grapheme_break_extended(boundclass, initial, &boundclass);
-	size_t start = 0;
-	size_t cpos = 0;
-	while(true) {
-		cpos += static_cast<size_t>(sz);
-		if (cpos >= len) {
-			fun(start, cpos);
-			return;
-		}
-		int next = utf8proc_get_property(utf8proc_codepoint(s + cpos, sz))->boundclass;
-		if (grapheme_break_extended(boundclass, next, &boundclass)) {
-			if (!fun(start, cpos)) {
-				return;
-			}
-			start = cpos;
-		}
-	}
-}
 
 /**
  * Given a codepoint `c`, return the codepoint of the corresponding
@@ -673,6 +629,18 @@ UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c);
  */
 UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
 
+/**
+ * Given a codepoint `c`, return `1` if the codepoint corresponds to a lower-case character
+ * and `0` otherwise.
+ */
+UTF8PROC_DLLEXPORT int utf8proc_islower(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint `c`, return `1` if the codepoint corresponds to an upper-case character
+ * and `0` otherwise.
+ */
+UTF8PROC_DLLEXPORT int utf8proc_isupper(utf8proc_int32_t c);
+
 /**
  * Given a codepoint, return a character width analogous to `wcwidth(codepoint)`,
  * except that a width of 0 is returned for non-printable codepoints
@@ -680,8 +648,8 @@ UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
  *
  * @note
  * If you want to check for particular types of non-printable characters,
- * (analogous to `isprint` or `iscntrl`), use @ref utf8proc_category. */
-    UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t codepoint);
+ * (analogous to `isprint` or `iscntrl`), use utf8proc_category(). */
+UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t codepoint);
 
 /**
  * Return the Unicode category for the codepoint (one of the
@@ -706,7 +674,7 @@ UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t codepoi
  * contain NULL characters with the string if `str` contained NULL
  * characters). Other flags in the `options` field are passed to the
  * functions defined above, and regarded as described.  See also
- * @ref utf8proc_map_custom to supply a custom codepoint transformation.
+ * utf8proc_map_custom() to supply a custom codepoint transformation.
  *
  * In case of success the length of the new string is returned,
  * otherwise a negative error code is returned.
@@ -719,7 +687,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
 );
 
 /**
- * Like @ref utf8proc_map, but also takes a `custom_func` mapping function
+ * Like utf8proc_map(), but also takes a `custom_func` mapping function
  * that is called on each codepoint in `str` before any other transformations
  * (along with a `custom_data` pointer that is passed through to `custom_func`).
  * The `custom_func` argument is ignored if it is `NULL`.
@@ -733,27 +701,27 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
  *
  * Returns a pointer to newly allocated memory of a NFD, NFC, NFKD, NFKC or
  * NFKC_Casefold normalized version of the null-terminated string `str`.  These
- * are shortcuts to calling @ref utf8proc_map with @ref UTF8PROC_NULLTERM
+ * are shortcuts to calling utf8proc_map() with @ref UTF8PROC_NULLTERM
  * combined with @ref UTF8PROC_STABLE and flags indicating the normalization.
  */
 /** @{ */
 /** NFD normalization (@ref UTF8PROC_DECOMPOSE). */
-UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str, utf8proc_ssize_t len);
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen);
 /** NFC normalization (@ref UTF8PROC_COMPOSE). */
-UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str, utf8proc_ssize_t len);
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen);
+/** remove accents from a string */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_remove_accents(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen);
 /** NFKD normalization (@ref UTF8PROC_DECOMPOSE and @ref UTF8PROC_COMPAT). */
-UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str, utf8proc_ssize_t len);
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen);
 /** NFKC normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */
-UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str, utf8proc_ssize_t len);
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen);
 /**
  * NFKC_Casefold normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT
  * and @ref UTF8PROC_CASEFOLD and @ref UTF8PROC_IGNORE).
  **/
-UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC_Casefold(const utf8proc_uint8_t *str, utf8proc_ssize_t len);
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC_Casefold(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen);
 /** @} */
 
-//#ifdef __cplusplus
-//}
-//#endif
 }
+
 #endif

package/src/duckdb/third_party/utf8proc/include/utf8proc_wrapper.hpp

@@ -14,6 +14,7 @@
 #include <cstdint>
 
 namespace duckdb {
+class GraphemeIterator;
 
 enum class UnicodeType { INVALID, ASCII, UNICODE };
 enum class UnicodeInvalidReason { BYTE_MISMATCH, INVALID_UNICODE };
@@ -43,6 +44,58 @@ public:
 	static size_t RenderWidth(const char *s, size_t len, size_t pos);
 	static size_t RenderWidth(const std::string &str);
 
+	static int32_t CodepointToUpper(int32_t codepoint);
+	static int32_t CodepointToLower(int32_t codepoint);
+
+	//! Constructs a class that can be iterated over to fetch grapheme clusters in a string
+	static GraphemeIterator GraphemeClusters(const char *s, size_t len);
+
+	//! Returns the number of grapheme clusters in a string
+	static size_t GraphemeCount(const char *s, size_t len);
+
+
 };
 
+struct GraphemeCluster {
+	size_t start;
+	size_t end;
+};
+
+class GraphemeIterator {
+public:
+	GraphemeIterator(const char *s, size_t len);
+
+private:
+	const char *s;
+	size_t len;
+
+private:
+	class GraphemeClusterIterator {
+	public:
+		GraphemeClusterIterator(const char *s, size_t len);
+
+		const char *s;
+		size_t len;
+		GraphemeCluster cluster;
+
+	public:
+		void Next();
+		void SetInvalid();
+		bool IsInvalid() const;
+
+		GraphemeClusterIterator &operator++();
+		bool operator!=(const GraphemeClusterIterator &other) const;
+		GraphemeCluster operator*() const;
+	};
+
+public:
+	GraphemeClusterIterator begin() { // NOLINT: match stl API
+		return GraphemeClusterIterator(s, len);
+	}
+	GraphemeClusterIterator end() { // NOLINT: match stl API
+		return GraphemeClusterIterator(nullptr, 0);
+	}
+};
+
+
 }