duckdb 1.1.4-dev9.0 → 1.2.0

package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp

@@ -34,6 +34,8 @@ public:
 //! yet in memory, and it only serializes dirty and non-written buffers to disk during
 //! serialization.
 class FixedSizeBuffer {
+	friend class FixedSizeAllocator;
+
 public:
 	//! Constants for fast offset calculations in the bitmask
 	static constexpr idx_t BASE[] = {0x00000000FFFFFFFF, 0x0000FFFF, 0x00FF, 0x0F, 0x3, 0x1};
@@ -46,33 +48,12 @@ public:
 	FixedSizeBuffer(BlockManager &block_manager, const idx_t segment_count, const idx_t allocation_size,
 	                const BlockPointer &block_pointer);
 
-	//! Block manager of the database instance
-	BlockManager &block_manager;
-
-	//! The number of allocated segments
-	idx_t segment_count;
-	//! The size of allocated memory in this buffer (necessary for copying while pinning)
-	idx_t allocation_size;
-
-	//! True: the in-memory buffer is no longer consistent with a (possibly existing) copy on disk
-	bool dirty;
-	//! True: can be vacuumed after the vacuum operation
-	bool vacuum;
-
-	//! Partial block id and offset
-	BlockPointer block_pointer;
+	~FixedSizeBuffer();
 
-public:
-	//! Returns true, if the buffer is in-memory
-	inline bool InMemory() const {
-		return buffer_handle.IsValid();
-	}
-	//! Returns true, if the block is on-disk
-	inline bool OnDisk() const {
-		return block_pointer.IsValid();
-	}
+private:
 	//! Returns a pointer to the buffer in memory, and calls Deserialize, if the buffer is not in memory
-	inline data_ptr_t Get(const bool dirty_p = true) {
+	data_ptr_t Get(const bool dirty_p = true) {
+		lock_guard<mutex> l(lock);
 		if (!InMemory()) {
 			Pin();
 		}
@@ -81,8 +62,17 @@ public:
 		}
 		return buffer_handle.Ptr();
 	}
-	//! Destroys the in-memory buffer and the on-disk block
-	void Destroy();
+
+	//! Returns true, if the buffer is in-memory
+	bool InMemory() const {
+		return buffer_handle.IsValid();
+	}
+
+	//! Returns true, if the block is on-disk
+	bool OnDisk() const {
+		return block_pointer.IsValid();
+	}
+
 	//! Serializes a buffer (if dirty or not on disk)
 	void Serialize(PartialBlockManager &partial_block_manager, const idx_t available_segments, const idx_t segment_size,
 	               const idx_t bitmask_offset);
@@ -92,17 +82,32 @@ public:
 	uint32_t GetOffset(const idx_t bitmask_count, const idx_t available_segments);
 	//! Sets the allocation size, if dirty
 	void SetAllocationSize(const idx_t available_segments, const idx_t segment_size, const idx_t bitmask_offset);
+	//! Sets all uninitialized regions of a buffer in the respective partial block allocation
+	void SetUninitializedRegions(PartialBlockForIndex &p_block_for_index, const idx_t segment_size, const idx_t offset,
+	                             const idx_t bitmask_offset, const idx_t available_segments);
 
 private:
+	//! Block manager of the database instance
+	BlockManager &block_manager;
+
+	//! The number of allocated segments
+	idx_t segment_count;
+	//! The size of allocated memory in this buffer (necessary for copying while pinning)
+	idx_t allocation_size;
+
+	//! True: the in-memory buffer is no longer consistent with a (possibly existing) copy on disk
+	bool dirty;
+	//! True: can be vacuumed after the vacuum operation
+	bool vacuum;
+
+	//! Partial block id and offset
+	BlockPointer block_pointer;
 	//! The buffer handle of the in-memory buffer
 	BufferHandle buffer_handle;
 	//! The block handle of the on-disk buffer
 	shared_ptr<BlockHandle> block_handle;
-
-private:
-	//! Sets all uninitialized regions of a buffer in the respective partial block allocation
-	void SetUninitializedRegions(PartialBlockForIndex &p_block_for_index, const idx_t segment_size, const idx_t offset,
-	                             const idx_t bitmask_offset, const idx_t available_segments);
+	//! The lock for this fixed size buffer handle
+	mutex lock;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp

@@ -17,6 +17,35 @@
 namespace duckdb {
 
 class CSVFileScan;
+
+//! Class that keeps track of line starts, used for line size verification
+class LinePosition {
+public:
+	LinePosition() {
+	}
+	LinePosition(idx_t buffer_idx_p, idx_t buffer_pos_p, idx_t buffer_size_p)
+	    : buffer_pos(buffer_pos_p), buffer_size(buffer_size_p), buffer_idx(buffer_idx_p) {
+	}
+
+	idx_t operator-(const LinePosition &other) const {
+		if (other.buffer_idx == buffer_idx) {
+			return buffer_pos - other.buffer_pos;
+		}
+		return other.buffer_size - other.buffer_pos + buffer_pos;
+	}
+
+	bool operator==(const LinePosition &other) const {
+		return buffer_pos == other.buffer_pos && buffer_idx == other.buffer_idx && buffer_size == other.buffer_size;
+	}
+
+	idx_t GetGlobalPosition(idx_t requested_buffer_size, bool first_char_nl = false) const {
+		return requested_buffer_size * buffer_idx + buffer_pos + first_char_nl;
+	}
+	idx_t buffer_pos = 0;
+	idx_t buffer_size = 0;
+	idx_t buffer_idx = 0;
+};
+
 class ScannerResult {
 public:
 	ScannerResult(CSVStates &states, CSVStateMachine &state_machine, idx_t result_size);
@@ -52,6 +81,10 @@ public:
 		return result.comment == true;
 	}
 
+	inline bool IsStateCurrent(CSVState state) const {
+		return states.states[1] == state;
+	}
+
 	//! Variable to keep information regarding quoted and escaped values
 	bool quoted = false;
 	//! If the current quoted value is unquoted
@@ -62,6 +95,8 @@ public:
 	bool comment = false;
 	idx_t quoted_position = 0;
 
+	LinePosition last_position;
+
 	//! Size of the result
 	const idx_t result_size;
 
@@ -88,7 +123,7 @@ public:
 	//! Returns true if the scanner is finished
 	bool FinishedFile() const;
 
-	//! Parses data into a output_chunk
+	//! Parses data into an output_chunk
 	virtual ScannerResult &ParseChunk();
 
 	//! Returns the result from the last Parse call. Shouts at you if you call it wrong

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp

@@ -41,6 +41,9 @@ public:
 	bool error = false;
 	idx_t result_position = 0;
 	bool cur_line_starts_as_comment = false;
+
+	idx_t cur_buffer_idx = 0;
+	idx_t current_buffer_size = 0;
 	//! How many rows fit a given column count
 	map<idx_t, idx_t> rows_per_column_count;
 	//! Adds a Value to the result

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp

@@ -138,6 +138,8 @@ public:
 
 	idx_t LinesSniffed() const;
 
+	bool EmptyOrOnlyHeader() const;
+
 private:
 	//! CSV State Machine Cache
 	CSVStateMachineCache &state_machine_cache;

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/state_machine_options.hpp

@@ -16,9 +16,9 @@ namespace duckdb {
 struct CSVStateMachineOptions {
 	CSVStateMachineOptions() {};
 	CSVStateMachineOptions(string delimiter_p, char quote_p, char escape_p, char comment_p,
-	                       NewLineIdentifier new_line_p, bool rfc_4180_p)
+	                       NewLineIdentifier new_line_p, bool strict_mode_p)
 	    : delimiter(std::move(delimiter_p)), quote(quote_p), escape(escape_p), comment(comment_p), new_line(new_line_p),
-	      rfc_4180(rfc_4180_p) {};
+	      strict_mode(strict_mode_p) {};
 
 	//! Delimiter to separate columns within each line
 	CSVOption<string> delimiter {","};
@@ -30,12 +30,12 @@ struct CSVStateMachineOptions {
 	CSVOption<char> comment = '\0';
 	//! New Line separator
 	CSVOption<NewLineIdentifier> new_line = NewLineIdentifier::NOT_SET;
-	//! RFC 4180 conformance
-	CSVOption<bool> rfc_4180 = false;
+	//! How Strict the parser should be
+	CSVOption<bool> strict_mode = true;
 
 	bool operator==(const CSVStateMachineOptions &other) const {
 		return delimiter == other.delimiter && quote == other.quote && escape == other.escape &&
-		       new_line == other.new_line && comment == other.comment && rfc_4180 == other.rfc_4180;
+		       new_line == other.new_line && comment == other.comment && strict_mode == other.strict_mode;
 	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp

@@ -27,34 +27,6 @@ struct CSVBufferUsage {
 	idx_t buffer_idx;
 };
 
-//! Class that keeps track of line starts, used for line size verification
-class LinePosition {
-public:
-	LinePosition() {
-	}
-	LinePosition(idx_t buffer_idx_p, idx_t buffer_pos_p, idx_t buffer_size_p)
-	    : buffer_pos(buffer_pos_p), buffer_size(buffer_size_p), buffer_idx(buffer_idx_p) {
-	}
-
-	idx_t operator-(const LinePosition &other) const {
-		if (other.buffer_idx == buffer_idx) {
-			return buffer_pos - other.buffer_pos;
-		}
-		return other.buffer_size - other.buffer_pos + buffer_pos;
-	}
-
-	bool operator==(const LinePosition &other) const {
-		return buffer_pos == other.buffer_pos && buffer_idx == other.buffer_idx && buffer_size == other.buffer_size;
-	}
-
-	idx_t GetGlobalPosition(idx_t requested_buffer_size, bool first_char_nl = false) const {
-		return requested_buffer_size * buffer_idx + buffer_pos + first_char_nl;
-	}
-	idx_t buffer_pos = 0;
-	idx_t buffer_size = 0;
-	idx_t buffer_idx = 0;
-};
-
 //! Keeps track of start and end of line positions in regard to the CSV file
 class FullLinePosition {
 public:
@@ -181,7 +153,7 @@ public:
 	unsafe_vector<ValidityMask *> validity_mask;
 
 	//! Variables to iterate over the CSV buffers
-	LinePosition last_position;
+
 	char *buffer_ptr;
 	idx_t buffer_size;
 	idx_t position_before_comment;
@@ -245,6 +217,8 @@ public:
 	//! (i.e., non-comment) line.
 	bool first_line_is_comment = false;
 
+	bool ignore_empty_values = true;
+
 	//! Specialized code for quoted values, makes sure to remove quotes and escapes
 	static inline void AddQuotedValue(StringValueResult &result, const idx_t buffer_pos);
 	//! Specialized code for possibly escaped values, makes sure to remove escapes
@@ -322,7 +296,8 @@ public:
 	bool FinishedIterator() const;
 
 	//! Creates a new string with all escaped values removed
-	static string_t RemoveEscape(const char *str_ptr, idx_t end, char escape, char quote, Vector &vector);
+	static string_t RemoveEscape(const char *str_ptr, idx_t end, char escape, char quote, bool strict_mode,
+	                             Vector &vector);
 
 	//! If we can directly cast the type when consuming the CSV file, or we have to do it later
 	static bool CanDirectlyCast(const LogicalType &type, bool icu_loaded);

package/src/duckdb/src/include/duckdb/execution/reservoir_sample.hpp

@@ -172,7 +172,13 @@ public:
 	static constexpr const SampleType TYPE = SampleType::RESERVOIR_SAMPLE;
 
 	constexpr static idx_t FIXED_SAMPLE_SIZE_MULTIPLIER = 10;
-	constexpr static idx_t FAST_TO_SLOW_THRESHOLD = 60;
+	// size is small enough, then the threshold to switch
+	// MinValue between std vec size and fixed sample size.
+	// During 'fast' sampling, we want every new vector to have the potential
+	// to add to the sample. If the threshold is too far below the standard vector size, then
+	// samples in the sample have a higher weight than new samples coming in.
+	// i.e during vector_size=2, 2 new samples will not be significant compared 2048 samples from 204800 tuples.
+	constexpr static idx_t FAST_TO_SLOW_THRESHOLD = MinValue<idx_t>(STANDARD_VECTOR_SIZE, 60);
 
 	// If the table has less than 204800 rows, this is the percentage
 	// of values we save when serializing/returning a sample.

package/src/duckdb/src/include/duckdb/function/scalar_function.hpp

@@ -78,13 +78,13 @@ struct FunctionModifiedDatabasesInput {
 
 struct FunctionBindExpressionInput {
 	FunctionBindExpressionInput(ClientContext &context_p, optional_ptr<FunctionData> bind_data_p,
-	                            BoundFunctionExpression &function_p)
-	    : context(context_p), bind_data(bind_data_p), function(function_p) {
+	                            vector<unique_ptr<Expression>> &children_p)
+	    : context(context_p), bind_data(bind_data_p), children(children_p) {
 	}
 
 	ClientContext &context;
 	optional_ptr<FunctionData> bind_data;
-	BoundFunctionExpression &function;
+	vector<unique_ptr<Expression>> &children;
 };
 
 struct ScalarFunctionBindInput {

package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_duck_schema.hpp

@@ -84,6 +84,7 @@ public:
 	}
 	void ThrowIfInvalid() const;
 
+	static unique_ptr<ArrowType> GetTypeFromFormat(string &format);
 	static unique_ptr<ArrowType> GetTypeFromFormat(DBConfig &config, ArrowSchema &schema, string &format);
 
 	static unique_ptr<ArrowType> GetTypeFromSchema(DBConfig &config, ArrowSchema &schema);

package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp

@@ -103,6 +103,10 @@ struct DuckDBOptimizersFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct DuckDBSecretTypesFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct DuckDBSequencesFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };

package/src/duckdb/src/include/duckdb/function/window/window_boundaries_state.hpp

@@ -113,9 +113,9 @@ struct WindowBoundariesState {
 	void ValidEnd(DataChunk &bounds, idx_t row_idx, const idx_t count, bool is_jump, const ValidityMask &partition_mask,
 	              const ValidityMask &order_mask, optional_ptr<WindowCursor> range);
 	void FrameBegin(DataChunk &bounds, idx_t row_idx, const idx_t count, WindowInputExpression &boundary_begin,
-	                optional_ptr<WindowCursor> range);
+	                const ValidityMask &order_mask, optional_ptr<WindowCursor> range);
 	void FrameEnd(DataChunk &bounds, idx_t row_idx, const idx_t count, WindowInputExpression &boundary_end,
-	              optional_ptr<WindowCursor> range);
+	              const ValidityMask &order_mask, optional_ptr<WindowCursor> range);
 
 	static void ClampFrame(const idx_t count, idx_t *values, const idx_t *begin, const idx_t *end) {
 		for (idx_t i = 0; i < count; ++i) {

package/src/duckdb/src/include/duckdb/logging/logger.hpp

@@ -25,6 +25,34 @@ class FileOpener;
 class LogStorage;
 class ExecutionContext;
 
+//! Logger Macro's are preferred method of calling logger
+#define DUCKDB_LOG(SOURCE, TYPE, LEVEL, ...)                                                                           \
+	{                                                                                                                  \
+		auto &logger = Logger::Get(SOURCE);                                                                            \
+		if (logger.ShouldLog(TYPE, LEVEL)) {                                                                           \
+			logger.WriteLog(TYPE, LEVEL, __VA_ARGS__);                                                                 \
+		}                                                                                                              \
+	}
+
+//! Use below macros to write to logger.
+// Parameters:
+//		SOURCE: the context to fetch the logger from, e.g. the ClientContext, or the DatabaseInstance, see Logger::Get
+//		TYPE  : a string describing the type of this log entry. Preferred format: `<duckdb/extension name>(.<sometype>)
+//				e.g. `duckdb.Extensions.ExtensionAutoloaded`, `my_extension` or `my_extension.some_type`
+//		PARAMS: Either a string-like type such as `const char *`, `string` or `string_t` or a format string plus the
+// 			string parameters
+//
+// Examples:
+//		DUCKDB_LOG_TRACE(client_context, "duckdb", "Something happened");
+//		DUCKDB_LOG_INFO(database_instance, "duckdb", CallFunctionThatReturnsString());
+//
+#define DUCKDB_LOG_TRACE(SOURCE, TYPE, ...) DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_TRACE, __VA_ARGS__)
+#define DUCKDB_LOG_DEBUG(SOURCE, TYPE, ...) DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_DEBUG, __VA_ARGS__)
+#define DUCKDB_LOG_INFO(SOURCE, TYPE, ...)  DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_INFO, __VA_ARGS__)
+#define DUCKDB_LOG_WARN(SOURCE, TYPE, ...)  DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_WARN, __VA_ARGS__)
+#define DUCKDB_LOG_ERROR(SOURCE, TYPE, ...) DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_ERROR, __VA_ARGS__)
+#define DUCKDB_LOG_FATAL(SOURCE, TYPE, ...) DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_FATAL, __VA_ARGS__)
+
 //! Main logging interface
 class Logger {
 public:
@@ -33,19 +61,20 @@ public:
 
 	DUCKDB_API virtual ~Logger() = default;
 
-	//! Main logger functions
-	DUCKDB_API void Log(const char *log_type, LogLevel log_level, const char *log_message);
-	DUCKDB_API void Log(LogLevel log_level, const char *log_message);
-	DUCKDB_API void Log(const char *log_type, LogLevel log_level, const string_t &log_message);
-	DUCKDB_API void Log(LogLevel log_level, const string_t &log_message);
-	DUCKDB_API void Log(const char *log_type, LogLevel log_level, std::function<string()> callback);
-	DUCKDB_API void Log(LogLevel log_level, std::function<string()> callback);
-
-	// Main interface for subclasses
+	// Main Logging interface. In most cases the macros above should be used instead of calling these directly
 	DUCKDB_API virtual bool ShouldLog(const char *log_type, LogLevel log_level) = 0;
-	DUCKDB_API virtual bool ShouldLog(LogLevel log_level) = 0;
 	DUCKDB_API virtual void WriteLog(const char *log_type, LogLevel log_level, const char *message) = 0;
-	DUCKDB_API virtual void WriteLog(LogLevel log_level, const char *message) = 0;
+
+	// Some more string types for easy logging
+	DUCKDB_API void WriteLog(const char *log_type, LogLevel log_level, const string &message);
+	DUCKDB_API void WriteLog(const char *log_type, LogLevel log_level, const string_t &message);
+
+	// Syntactic sugar for formatted strings
+	template <typename... ARGS>
+	void WriteLog(const char *log_type, LogLevel log_level, const char *format_string, ARGS... params) {
+		auto formatted_string = StringUtil::Format(format_string, params...);
+		WriteLog(log_type, log_level, formatted_string.c_str());
+	}
 
 	DUCKDB_API virtual void Flush() = 0;
 
@@ -58,104 +87,6 @@ public:
 	DUCKDB_API static Logger &Get(const FileOpener &opener);
 	DUCKDB_API static Logger &Get(const DatabaseInstance &db);
 
-	//! Logger::Log with raw C-String
-	template <class T>
-	static void Log(const char *log_type, T &log_context_source, LogLevel log_level, const char *log_message) {
-		Logger::Get(log_context_source).Log(log_type, log_level, log_message);
-	}
-	template <class T>
-	static void Log(const char *log_type, T &log_context_source, LogLevel log_level, const string &log_message) {
-		Logger::Get(log_context_source).Log(log_type, log_level, log_message.c_str());
-	}
-	template <class T>
-	static void Log(const char *log_type, T &log_context_source, LogLevel log_level, const string_t &log_message) {
-		Logger::Get(log_context_source).Log(log_type, log_level, log_message);
-	}
-	template <class T>
-	static void Log(T &log_context_source, LogLevel log_level, const char *log_message) {
-		Logger::Get(log_context_source).Log(log_level, log_message);
-	}
-	template <class T>
-	static void Log(T &log_context_source, LogLevel log_level, const string &log_message) {
-		Logger::Get(log_context_source).Log(log_level, log_message.c_str());
-	}
-	template <class T>
-	static void Log(T &log_context_source, LogLevel log_level, const string_t &log_message) {
-		Logger::Get(log_context_source).Log(log_level, log_message);
-	}
-
-	//! Logger::Log with callback
-	template <class T>
-	static void Log(const char *log_type, T &log_context_source, LogLevel log_level, std::function<string()> callback) {
-		Logger::Get(log_context_source).Log(log_type, log_level, std::move(callback));
-	}
-	template <class T>
-	static void Log(T &log_context_source, LogLevel log_level, std::function<string()> callback) {
-		Logger::Get(log_context_source).Log(log_level, std::move(callback));
-	}
-	//! Logger::Log with StringUtil::Format
-	template <class T, typename... ARGS>
-	static void Log(const char *log_type, T &log_context_source, LogLevel log_level, const char *format_string,
-	                ARGS... params) {
-		Logger::Get(log_context_source).Log(log_type, log_level, [&]() {
-			return StringUtil::Format(format_string, params...);
-		});
-	}
-	template <class T, typename... ARGS>
-	static void Log(T &log_context_source, LogLevel log_level, const char *format_string, ARGS... params) {
-		Logger::Get(log_context_source).Log(log_level, [&]() { return StringUtil::Format(format_string, params...); });
-	}
-
-	//! Templates wrapping Logging::Log(..., LoggingLevel, ....)
-	template <class T, typename... ARGS>
-	static void Trace(T &log_context_source, ARGS... params) {
-		Log(log_context_source, LogLevel::LOG_TRACE, params...);
-	}
-	template <class T, typename... ARGS>
-	static void Trace(const char *log_type, T &log_context_source, ARGS... params) {
-		Log(log_type, log_context_source, LogLevel::LOG_TRACE, params...);
-	}
-	template <class T, typename... ARGS>
-	static void Debug(T &log_context_source, ARGS... params) {
-		Log(log_context_source, LogLevel::LOG_DEBUG, params...);
-	}
-	template <class T, typename... ARGS>
-	static void Debug(const char *log_type, T &log_context_source, ARGS... params) {
-		Log(log_type, log_context_source, LogLevel::LOG_DEBUG, params...);
-	}
-	template <class T, typename... ARGS>
-	static void Info(T &log_context_source, ARGS... params) {
-		Log(log_context_source, LogLevel::LOG_INFO, params...);
-	}
-	template <class T, typename... ARGS>
-	static void Info(const char *log_type, T &log_context_source, ARGS... params) {
-		Log(log_type, log_context_source, LogLevel::LOG_INFO, params...);
-	}
-	template <class T, typename... ARGS>
-	static void Warn(T &log_context_source, ARGS... params) {
-		Log(log_context_source, LogLevel::LOG_WARN, params...);
-	}
-	template <class T, typename... ARGS>
-	static void Warn(const char *log_type, T &log_context_source, ARGS... params) {
-		Log(log_type, log_context_source, LogLevel::LOG_WARN, params...);
-	}
-	template <class T, typename... ARGS>
-	static void Error(T &log_context_source, ARGS... params) {
-		Log(log_context_source, LogLevel::LOG_ERROR, params...);
-	}
-	template <class T, typename... ARGS>
-	static void Error(const char *log_type, T &log_context_source, ARGS... params) {
-		Log(log_type, log_context_source, LogLevel::LOG_ERROR, params...);
-	}
-	template <class T, typename... ARGS>
-	static void Fatal(T &log_context_source, ARGS... params) {
-		Log(log_context_source, LogLevel::LOG_FATAL, params...);
-	}
-	template <class T, typename... ARGS>
-	static void Fatal(const char *log_type, T &log_context_source, ARGS... params) {
-		Log(log_type, log_context_source, LogLevel::LOG_FATAL, params...);
-	}
-
 	template <class T>
 	static void Flush(T &log_context_source) {
 		Get(log_context_source).Flush();
@@ -182,9 +113,7 @@ public:
 
 	// Main Logger API
 	bool ShouldLog(const char *log_type, LogLevel log_level) override;
-	bool ShouldLog(LogLevel log_level) override;
 	void WriteLog(const char *log_type, LogLevel log_level, const char *message) override;
-	void WriteLog(LogLevel log_level, const char *message) override;
 
 	void Flush() override;
 	bool IsThreadSafe() override {
@@ -209,9 +138,7 @@ public:
 
 	// Main Logger API
 	bool ShouldLog(const char *log_type, LogLevel log_level) override;
-	bool ShouldLog(LogLevel log_level) override;
 	void WriteLog(const char *log_type, LogLevel log_level, const char *message) override;
-	void WriteLog(LogLevel log_level, const char *message) override;
 	void Flush() override;
 
 	bool IsThreadSafe() override {
@@ -234,9 +161,7 @@ public:
 
 	// Main Logger API
 	bool ShouldLog(const char *log_type, LogLevel log_level) override;
-	bool ShouldLog(LogLevel log_level) override;
 	void WriteLog(const char *log_type, LogLevel log_level, const char *message) override;
-	void WriteLog(LogLevel log_level, const char *message) override;
 
 	void Flush() override;
 	bool IsThreadSafe() override {
@@ -269,11 +194,7 @@ public:
 	bool ShouldLog(const char *log_type, LogLevel log_level) override {
 		return false;
 	}
-	bool ShouldLog(LogLevel log_level) override {
-		return false;
-	};
 	void WriteLog(const char *log_type, LogLevel log_level, const char *message) override {};
-	void WriteLog(LogLevel log_level, const char *message) override {};
 	void Flush() override {
 	}
 	bool IsThreadSafe() override {

package/src/duckdb/src/include/duckdb/logging/logging.hpp

@@ -71,8 +71,6 @@ struct LoggingContext {
 	optional_idx thread;
 	optional_idx client_context;
 	optional_idx transaction_id;
-
-	const char *default_log_type = "default";
 };
 
 struct RegisteredLoggingContext {

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -101,6 +101,8 @@ struct ExtensionOption {
 
 class SerializationCompatibility {
 public:
+	static SerializationCompatibility FromDatabase(AttachedDatabase &db);
+	static SerializationCompatibility FromIndex(idx_t serialization_version);
 	static SerializationCompatibility FromString(const string &input);
 	static SerializationCompatibility Default();
 	static SerializationCompatibility Latest();
@@ -202,6 +204,8 @@ struct DBConfigOptions {
 	uint64_t zstd_min_string_length = 4096;
 	//! Force a specific compression method to be used when checkpointing (if available)
 	CompressionType force_compression = CompressionType::COMPRESSION_AUTO;
+	//! The set of disabled compression methods (default empty)
+	set<CompressionType> disabled_compression_methods;
 	//! Force a specific bitpacking mode to be used when using the bitpacking compression method
 	BitpackingMode force_bitpacking_mode = BitpackingMode::AUTO;
 	//! Debug setting for window aggregation mode: (window, combine, separate)
@@ -380,6 +384,7 @@ public:
 	DUCKDB_API ArrowTypeExtension GetArrowExtension(ArrowExtensionMetadata info) const;
 	DUCKDB_API ArrowTypeExtension GetArrowExtension(const LogicalType &type) const;
 	DUCKDB_API bool HasArrowExtension(const LogicalType &type) const;
+	DUCKDB_API bool HasArrowExtension(ArrowExtensionMetadata info) const;
 	DUCKDB_API void RegisterArrowExtension(const ArrowTypeExtension &extension) const;
 
 	bool operator==(const DBConfig &other);

package/src/duckdb/src/include/duckdb/main/connection.hpp

@@ -166,14 +166,6 @@ public:
 	DUCKDB_API shared_ptr<Relation> RelationFromQuery(unique_ptr<SelectStatement> select_stmt,
 	                                                  const string &alias = "queryrelation", const string &query = "");
 
-	//! Returns a substrait BLOB from a valid query
-	DUCKDB_API string GetSubstrait(const string &query);
-	//! Returns a Query Result from a substrait blob
-	DUCKDB_API unique_ptr<QueryResult> FromSubstrait(const string &proto);
-	//! Returns a substrait BLOB from a valid query
-	DUCKDB_API string GetSubstraitJSON(const string &query);
-	//! Returns a Query Result from a substrait JSON
-	DUCKDB_API unique_ptr<QueryResult> FromSubstraitJSON(const string &json);
 	DUCKDB_API void BeginTransaction();
 	DUCKDB_API void Commit();
 	DUCKDB_API void Rollback();

package/src/duckdb/src/include/duckdb/main/connection_manager.hpp

@@ -34,8 +34,9 @@ public:
 	static ConnectionManager &Get(ClientContext &context);
 
 private:
-	mutable mutex connections_lock;
+	mutex connections_lock;
 	reference_map_t<ClientContext, weak_ptr<ClientContext>> connections;
+	atomic<idx_t> connection_count;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/extension.hpp

@@ -24,6 +24,7 @@ public:
 	DUCKDB_API virtual std::string Version() const {
 		return "";
 	}
+	DUCKDB_API static const char *DefaultVersion();
 };
 
 enum class ExtensionABIType : uint8_t {