duckdb 0.8.1-dev96.0 → 0.8.1

package/src/duckdb/src/catalog/default/default_types.cpp

@@ -4,94 +4,18 @@
 #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/parser/parsed_data/create_type_info.hpp"
+#include "duckdb/catalog/default/builtin_types/types.hpp"
 
 namespace duckdb {
 
-struct DefaultType {
-	const char *name;
-	LogicalTypeId type;
-};
-
-static DefaultType internal_types[] = {{"int", LogicalTypeId::INTEGER},
-                                       {"int4", LogicalTypeId::INTEGER},
-                                       {"signed", LogicalTypeId::INTEGER},
-                                       {"integer", LogicalTypeId::INTEGER},
-                                       {"integral", LogicalTypeId::INTEGER},
-                                       {"int32", LogicalTypeId::INTEGER},
-                                       {"varchar", LogicalTypeId::VARCHAR},
-                                       {"bpchar", LogicalTypeId::VARCHAR},
-                                       {"text", LogicalTypeId::VARCHAR},
-                                       {"string", LogicalTypeId::VARCHAR},
-                                       {"char", LogicalTypeId::VARCHAR},
-                                       {"nvarchar", LogicalTypeId::VARCHAR},
-                                       {"bytea", LogicalTypeId::BLOB},
-                                       {"blob", LogicalTypeId::BLOB},
-                                       {"varbinary", LogicalTypeId::BLOB},
-                                       {"binary", LogicalTypeId::BLOB},
-                                       {"bit", LogicalTypeId::BIT},
-                                       {"bitstring", LogicalTypeId::BIT},
-                                       {"int8", LogicalTypeId::BIGINT},
-                                       {"bigint", LogicalTypeId::BIGINT},
-                                       {"int64", LogicalTypeId::BIGINT},
-                                       {"long", LogicalTypeId::BIGINT},
-                                       {"oid", LogicalTypeId::BIGINT},
-                                       {"int2", LogicalTypeId::SMALLINT},
-                                       {"smallint", LogicalTypeId::SMALLINT},
-                                       {"short", LogicalTypeId::SMALLINT},
-                                       {"int16", LogicalTypeId::SMALLINT},
-                                       {"timestamp", LogicalTypeId::TIMESTAMP},
-                                       {"datetime", LogicalTypeId::TIMESTAMP},
-                                       {"timestamp_us", LogicalTypeId::TIMESTAMP},
-                                       {"timestamp_ms", LogicalTypeId::TIMESTAMP_MS},
-                                       {"timestamp_ns", LogicalTypeId::TIMESTAMP_NS},
-                                       {"timestamp_s", LogicalTypeId::TIMESTAMP_SEC},
-                                       {"bool", LogicalTypeId::BOOLEAN},
-                                       {"boolean", LogicalTypeId::BOOLEAN},
-                                       {"logical", LogicalTypeId::BOOLEAN},
-                                       {"decimal", LogicalTypeId::DECIMAL},
-                                       {"dec", LogicalTypeId::DECIMAL},
-                                       {"numeric", LogicalTypeId::DECIMAL},
-                                       {"real", LogicalTypeId::FLOAT},
-                                       {"float4", LogicalTypeId::FLOAT},
-                                       {"float", LogicalTypeId::FLOAT},
-                                       {"double", LogicalTypeId::DOUBLE},
-                                       {"float8", LogicalTypeId::DOUBLE},
-                                       {"tinyint", LogicalTypeId::TINYINT},
-                                       {"int1", LogicalTypeId::TINYINT},
-                                       {"date", LogicalTypeId::DATE},
-                                       {"time", LogicalTypeId::TIME},
-                                       {"interval", LogicalTypeId::INTERVAL},
-                                       {"hugeint", LogicalTypeId::HUGEINT},
-                                       {"int128", LogicalTypeId::HUGEINT},
-                                       {"uuid", LogicalTypeId::UUID},
-                                       {"guid", LogicalTypeId::UUID},
-                                       {"struct", LogicalTypeId::STRUCT},
-                                       {"row", LogicalTypeId::STRUCT},
-                                       {"list", LogicalTypeId::LIST},
-                                       {"map", LogicalTypeId::MAP},
-                                       {"utinyint", LogicalTypeId::UTINYINT},
-                                       {"uint8", LogicalTypeId::UTINYINT},
-                                       {"usmallint", LogicalTypeId::USMALLINT},
-                                       {"uint16", LogicalTypeId::USMALLINT},
-                                       {"uinteger", LogicalTypeId::UINTEGER},
-                                       {"uint32", LogicalTypeId::UINTEGER},
-                                       {"ubigint", LogicalTypeId::UBIGINT},
-                                       {"uint64", LogicalTypeId::UBIGINT},
-                                       {"union", LogicalTypeId::UNION},
-                                       {"timestamptz", LogicalTypeId::TIMESTAMP_TZ},
-                                       {"timetz", LogicalTypeId::TIME_TZ},
-                                       {"enum", LogicalTypeId::ENUM},
-                                       {"null", LogicalTypeId::SQLNULL},
-                                       {nullptr, LogicalTypeId::INVALID}};
-
 LogicalTypeId DefaultTypeGenerator::GetDefaultType(const string &name) {
-	auto lower_str = StringUtil::Lower(name);
-	for (idx_t index = 0; internal_types[index].name != nullptr; index++) {
-		if (internal_types[index].name == lower_str) {
-			return internal_types[index].type;
+	auto &internal_types = BUILTIN_TYPES;
+	for (auto &type : internal_types) {
+		if (StringUtil::CIEquals(name, type.name)) {
+			return type.type;
 		}
 	}
-	return LogicalTypeId::INVALID;
+	return LogicalType::INVALID;
 }
 
 DefaultTypeGenerator::DefaultTypeGenerator(Catalog &catalog, SchemaCatalogEntry &schema)
@@ -119,8 +43,9 @@ vector<string> DefaultTypeGenerator::GetDefaultEntries() {
 	if (schema.name != DEFAULT_SCHEMA) {
 		return result;
 	}
-	for (idx_t index = 0; internal_types[index].name != nullptr; index++) {
-		result.emplace_back(internal_types[index].name);
+	auto &internal_types = BUILTIN_TYPES;
+	for (auto &type : internal_types) {
+		result.emplace_back(StringUtil::Lower(type.name));
 	}
 	return result;
 }

package/src/duckdb/src/common/adbc/adbc.cpp

@@ -5,10 +5,13 @@
 #include "duckdb/common/string_util.hpp"
 
 #include "duckdb.h"
-#include "duckdb/main/connection.hpp"
 #include "duckdb/common/arrow/arrow_wrapper.hpp"
 #include "duckdb/common/arrow/arrow.hpp"
 
+#ifndef DUCKDB_AMALGAMATION
+#include "duckdb/main/connection.hpp"
+#endif
+
 #include <string.h>
 #include <stdlib.h>
 
@@ -37,6 +40,10 @@ duckdb_adbc::AdbcStatusCode duckdb_adbc_init(size_t count, struct duckdb_adbc::A
 	driver->StatementSetOption = duckdb_adbc::StatementSetOption;
 	driver->StatementSetSqlQuery = duckdb_adbc::StatementSetSqlQuery;
 	driver->ConnectionGetObjects = duckdb_adbc::ConnectionGetObjects;
+	driver->ConnectionCommit = duckdb_adbc::ConnectionCommit;
+	driver->ConnectionRollback = duckdb_adbc::ConnectionRollback;
+	driver->ConnectionReadPartition = duckdb_adbc::ConnectionReadPartition;
+	driver->StatementExecutePartitions = duckdb_adbc::StatementExecutePartitions;
 	return ADBC_STATUS_OK;
 }
 
@@ -162,10 +169,104 @@ AdbcStatusCode ConnectionNew(struct AdbcConnection *connection, struct AdbcError
 	return ADBC_STATUS_OK;
 }
 
+AdbcStatusCode ExecuteQuery(duckdb::Connection *conn, const char *query, struct AdbcError *error) {
+	auto res = conn->Query(query);
+	if (res->HasError()) {
+		auto error_message = "Failed to execute query \"" + std::string(query) + "\": " + res->GetError();
+		SetError(error, error_message);
+		return ADBC_STATUS_INTERNAL;
+	}
+	return ADBC_STATUS_OK;
+}
+
 AdbcStatusCode ConnectionSetOption(struct AdbcConnection *connection, const char *key, const char *value,
                                    struct AdbcError *error) {
-	// there are no connection-level options that need to be set before connecting
-	return ADBC_STATUS_OK;
+	if (!connection) {
+		SetError(error, "Connection is not set");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	}
+	auto conn = (duckdb::Connection *)connection->private_data;
+	if (strcmp(key, ADBC_CONNECTION_OPTION_AUTOCOMMIT) == 0) {
+		if (strcmp(value, ADBC_OPTION_VALUE_ENABLED) == 0) {
+			if (conn->HasActiveTransaction()) {
+				AdbcStatusCode status = ExecuteQuery(conn, "COMMIT", error);
+				if (status != ADBC_STATUS_OK) {
+					return status;
+				}
+			} else {
+				// no-op
+			}
+		} else if (strcmp(value, ADBC_OPTION_VALUE_DISABLED) == 0) {
+			if (conn->HasActiveTransaction()) {
+				// no-op
+			} else {
+				// begin
+				AdbcStatusCode status = ExecuteQuery(conn, "START TRANSACTION", error);
+				if (status != ADBC_STATUS_OK) {
+					return status;
+				}
+			}
+		} else {
+			auto error_message = "Invalid connection option value " + std::string(key) + "=" + std::string(value);
+			SetError(error, error_message);
+			return ADBC_STATUS_INVALID_ARGUMENT;
+		}
+		return ADBC_STATUS_OK;
+	}
+	auto error_message =
+	    "Unknown connection option " + std::string(key) + "=" + (value ? std::string(value) : "(NULL)");
+	SetError(error, error_message);
+	return ADBC_STATUS_NOT_IMPLEMENTED;
+}
+
+AdbcStatusCode ConnectionReadPartition(struct AdbcConnection *connection, const uint8_t *serialized_partition,
+                                       size_t serialized_length, struct ArrowArrayStream *out,
+                                       struct AdbcError *error) {
+	SetError(error, "Read Partitions are not supported in DuckDB");
+	return ADBC_STATUS_NOT_IMPLEMENTED;
+}
+
+AdbcStatusCode StatementExecutePartitions(struct AdbcStatement *statement, struct ArrowSchema *schema,
+                                          struct AdbcPartitions *partitions, int64_t *rows_affected,
+                                          struct AdbcError *error) {
+	SetError(error, "Execute Partitions are not supported in DuckDB");
+	return ADBC_STATUS_NOT_IMPLEMENTED;
+}
+
+AdbcStatusCode ConnectionCommit(struct AdbcConnection *connection, struct AdbcError *error) {
+	if (!connection) {
+		SetError(error, "Connection is not set");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	}
+	auto conn = (duckdb::Connection *)connection->private_data;
+	if (!conn->HasActiveTransaction()) {
+		SetError(error, "No active transaction, cannot commit");
+		return ADBC_STATUS_INVALID_STATE;
+	}
+
+	AdbcStatusCode status = ExecuteQuery(conn, "COMMIT", error);
+	if (status != ADBC_STATUS_OK) {
+		return status;
+	}
+	return ExecuteQuery(conn, "START TRANSACTION", error);
+}
+
+AdbcStatusCode ConnectionRollback(struct AdbcConnection *connection, struct AdbcError *error) {
+	if (!connection) {
+		SetError(error, "Connection is not set");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	}
+	auto conn = (duckdb::Connection *)connection->private_data;
+	if (!conn->HasActiveTransaction()) {
+		SetError(error, "No active transaction, cannot rollback");
+		return ADBC_STATUS_INVALID_STATE;
+	}
+
+	AdbcStatusCode status = ExecuteQuery(conn, "ROLLBACK", error);
+	if (status != ADBC_STATUS_OK) {
+		return status;
+	}
+	return ExecuteQuery(conn, "START TRANSACTION", error);
 }
 
 AdbcStatusCode ConnectionInit(struct AdbcConnection *connection, struct AdbcDatabase *database,
@@ -219,11 +320,11 @@ void release(struct ArrowArrayStream *stream) {
 	if (!stream || !stream->release) {
 		return;
 	}
-	stream->release = nullptr;
 	if (stream->private_data) {
 		duckdb_destroy_arrow((duckdb_arrow *)&stream->private_data);
 		stream->private_data = nullptr;
 	}
+	stream->release = nullptr;
 }
 
 const char *get_last_error(struct ArrowArrayStream *stream) {
@@ -270,16 +371,21 @@ AdbcStatusCode Ingest(duckdb_connection connection, const char *table_name, stru
 	if (status != ADBC_STATUS_OK) {
 		return status;
 	}
+	auto cconn = (duckdb::Connection *)connection;
 
+	auto has_table = cconn->TableInfo(table_name);
+	auto arrow_scan = cconn->TableFunction("arrow_scan", {duckdb::Value::POINTER((uintptr_t)input),
+	                                                      duckdb::Value::POINTER((uintptr_t)stream_produce),
+	                                                      duckdb::Value::POINTER((uintptr_t)get_schema)});
 	try {
-		// TODO evil cast, do we need a way to do this from the C api?
-		auto cconn = (duckdb::Connection *)connection;
-		cconn
-		    ->TableFunction("arrow_scan",
-		                    {duckdb::Value::POINTER((uintptr_t)input),
-		                     duckdb::Value::POINTER((uintptr_t)stream_produce),
-		                     duckdb::Value::POINTER((uintptr_t)get_schema)}) // TODO make this a parameter somewhere
-		    ->Create(table_name); // TODO this should probably be a temp table
+		if (!has_table) {
+			// We create the table based on an Arrow Scanner
+			arrow_scan->Create(table_name);
+		} else {
+			arrow_scan->CreateView("temp_adbc_view", true, true);
+			auto query = "insert into " + std::string(table_name) + " select * from temp_adbc_view";
+			auto result = cconn->Query(query);
+		}
 		// After creating a table, the arrow array stream is released. Hence we must set it as released to avoid
 		// double-releasing it
 		input->release = nullptr;

package/src/duckdb/src/common/adbc/driver_manager.cpp

@@ -133,10 +133,6 @@ static AdbcStatusCode ReleaseDriver(struct AdbcDriver *driver, struct AdbcError
 
 // Default stubs
 
-AdbcStatusCode ConnectionCommit(struct AdbcConnection *, struct AdbcError *error) {
-	return ADBC_STATUS_NOT_IMPLEMENTED;
-}
-
 AdbcStatusCode ConnectionGetInfo(struct AdbcConnection *connection, uint32_t *info_codes, size_t info_codes_length,
                                  struct ArrowArrayStream *out, struct AdbcError *error) {
 	return ADBC_STATUS_NOT_IMPLEMENTED;
@@ -147,27 +143,11 @@ AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *, const char *, c
 	return ADBC_STATUS_NOT_IMPLEMENTED;
 }
 
-AdbcStatusCode ConnectionReadPartition(struct AdbcConnection *connection, const uint8_t *serialized_partition,
-                                       size_t serialized_length, struct ArrowArrayStream *out,
-                                       struct AdbcError *error) {
-	return ADBC_STATUS_NOT_IMPLEMENTED;
-}
-
-AdbcStatusCode ConnectionRollback(struct AdbcConnection *, struct AdbcError *error) {
-	return ADBC_STATUS_NOT_IMPLEMENTED;
-}
-
 AdbcStatusCode StatementBind(struct AdbcStatement *, struct ArrowArray *, struct ArrowSchema *,
                              struct AdbcError *error) {
 	return ADBC_STATUS_NOT_IMPLEMENTED;
 }
 
-AdbcStatusCode StatementExecutePartitions(struct AdbcStatement *statement, struct ArrowSchema *schema,
-                                          struct AdbcPartitions *partitions, int64_t *rows_affected,
-                                          struct AdbcError *error) {
-	return ADBC_STATUS_NOT_IMPLEMENTED;
-}
-
 AdbcStatusCode StatementGetParameterSchema(struct AdbcStatement *statement, struct ArrowSchema *schema,
                                            struct AdbcError *error) {
 	return ADBC_STATUS_NOT_IMPLEMENTED;

package/src/duckdb/src/common/arrow/arrow_converter.cpp

@@ -16,7 +16,7 @@
 namespace duckdb {
 
 void ArrowConverter::ToArrowArray(DataChunk &input, ArrowArray *out_array, ArrowOptions options) {
-	ArrowAppender appender(input.GetTypes(), input.size(), options);
+	ArrowAppender appender(input.GetTypes(), input.size(), std::move(options));
 	appender.Append(input, 0, input.size(), input.size());
 	*out_array = appender.Finalize();
 }
@@ -59,10 +59,10 @@ void InitializeChild(ArrowSchema &child, const string &name = "") {
 	child.dictionary = nullptr;
 }
 void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                    const string &config_timezone, ArrowOptions options);
+                    const ArrowOptions &options);
 
 void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                       const string &config_timezone, ArrowOptions options) {
+                       const ArrowOptions &options) {
 	child.format = "+m";
 	//! Map has one child which is a struct
 	child.n_children = 1;
@@ -73,11 +73,11 @@ void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child,
 	InitializeChild(root_holder.nested_children.back()[0]);
 	child.children = &root_holder.nested_children_ptr.back()[0];
 	child.children[0]->name = "entries";
-	SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), config_timezone, options);
+	SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), options);
 }
 
 void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                    const string &config_timezone, ArrowOptions options) {
+                    const ArrowOptions &options) {
 	switch (type.id()) {
 	case LogicalTypeId::BOOLEAN:
 		child.format = "b";
@@ -134,7 +134,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		child.format = "tsu:";
 		break;
 	case LogicalTypeId::TIMESTAMP_TZ: {
-		string format = "tsu:" + config_timezone;
+		string format = "tsu:" + options.time_zone;
 		auto format_ptr = make_unsafe_uniq_array<char>(format.size() + 1);
 		for (size_t i = 0; i < format.size(); i++) {
 			format_ptr[i] = format[i];
@@ -192,7 +192,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		InitializeChild(root_holder.nested_children.back()[0]);
 		child.children = &root_holder.nested_children_ptr.back()[0];
 		child.children[0]->name = "l";
-		SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), config_timezone, options);
+		SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), options);
 		break;
 	}
 	case LogicalTypeId::STRUCT: {
@@ -220,13 +220,12 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 			root_holder.owned_type_names.push_back(std::move(name_ptr));
 
 			child.children[type_idx]->name = root_holder.owned_type_names.back().get();
-			SetArrowFormat(root_holder, *child.children[type_idx], child_types[type_idx].second, config_timezone,
-			               options);
+			SetArrowFormat(root_holder, *child.children[type_idx], child_types[type_idx].second, options);
 		}
 		break;
 	}
 	case LogicalTypeId::MAP: {
-		SetArrowMapFormat(root_holder, child, type, config_timezone, options);
+		SetArrowMapFormat(root_holder, child, type, options);
 		break;
 	}
 	case LogicalTypeId::ENUM: {
@@ -259,7 +258,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 }
 
 void ArrowConverter::ToArrowSchema(ArrowSchema *out_schema, const vector<LogicalType> &types,
-                                   const vector<string> &names, const string &config_timezone, ArrowOptions options) {
+                                   const vector<string> &names, const ArrowOptions &options) {
 	D_ASSERT(out_schema);
 	D_ASSERT(types.size() == names.size());
 	idx_t column_count = types.size();
@@ -287,7 +286,7 @@ void ArrowConverter::ToArrowSchema(ArrowSchema *out_schema, const vector<Logical
 
 		auto &child = root_holder->children[col_idx];
 		InitializeChild(child, names[col_idx]);
-		SetArrowFormat(*root_holder, child, types[col_idx], config_timezone, options);
+		SetArrowFormat(*root_holder, child, types[col_idx], options);
 	}
 
 	// Release ownership to caller

package/src/duckdb/src/common/arrow/arrow_wrapper.cpp

@@ -80,7 +80,7 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 	auto my_stream = reinterpret_cast<ResultArrowArrayStreamWrapper *>(stream->private_data);
 	if (!my_stream->column_types.empty()) {
 		ArrowConverter::ToArrowSchema(out, my_stream->column_types, my_stream->column_names,
-		                              my_stream->timezone_config);
+		                              QueryResult::GetArrowOptions(*my_stream->result));
 		return 0;
 	}
 
@@ -100,7 +100,8 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 		my_stream->column_types = result.types;
 		my_stream->column_names = result.names;
 	}
-	ArrowConverter::ToArrowSchema(out, my_stream->column_types, my_stream->column_names, my_stream->timezone_config);
+	ArrowConverter::ToArrowSchema(out, my_stream->column_types, my_stream->column_names,
+	                              QueryResult::GetArrowOptions(*my_stream->result));
 	return 0;
 }
 
@@ -186,7 +187,7 @@ bool ArrowUtil::TryFetchNext(QueryResult &result, unique_ptr<DataChunk> &chunk,
 bool ArrowUtil::TryFetchChunk(QueryResult *result, idx_t chunk_size, ArrowArray *out, idx_t &count,
                               PreservedError &error) {
 	count = 0;
-	ArrowAppender appender(result->types, chunk_size);
+	ArrowAppender appender(result->types, chunk_size, QueryResult::GetArrowOptions(*result));
 	auto &current_chunk = result->current_chunk;
 	if (current_chunk.Valid()) {
 		// We start by scanning the non-finished current chunk

package/src/duckdb/src/common/exception.cpp

@@ -73,8 +73,11 @@ string Exception::ConstructMessageRecursive(const string &msg, std::vector<Excep
 		parameter_count++;
 	}
 	if (parameter_count != values.size()) {
-		throw InternalException("Expected %d parameters, received %d", parameter_count, values.size());
+		throw InternalException("Primary exception: %s\nSecondary exception in ConstructMessageRecursive: Expected %d "
+		                        "parameters, received %d",
+		                        msg.c_str(), parameter_count, values.size());
 	}
+
 #endif
 	return ExceptionFormatValue::Format(msg, values);
 }

package/src/duckdb/src/common/exception_format_value.cpp

@@ -68,22 +68,31 @@ ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(hugeint_t value) {
 }
 
 string ExceptionFormatValue::Format(const string &msg, std::vector<ExceptionFormatValue> &values) {
-	std::vector<duckdb_fmt::basic_format_arg<duckdb_fmt::printf_context>> format_args;
-	for (auto &val : values) {
-		switch (val.type) {
-		case ExceptionFormatValueType::FORMAT_VALUE_TYPE_DOUBLE:
-			format_args.push_back(duckdb_fmt::internal::make_arg<duckdb_fmt::printf_context>(val.dbl_val));
-			break;
-		case ExceptionFormatValueType::FORMAT_VALUE_TYPE_INTEGER:
-			format_args.push_back(duckdb_fmt::internal::make_arg<duckdb_fmt::printf_context>(val.int_val));
-			break;
-		case ExceptionFormatValueType::FORMAT_VALUE_TYPE_STRING:
-			format_args.push_back(duckdb_fmt::internal::make_arg<duckdb_fmt::printf_context>(val.str_val));
-			break;
+	try {
+		std::vector<duckdb_fmt::basic_format_arg<duckdb_fmt::printf_context>> format_args;
+		for (auto &val : values) {
+			switch (val.type) {
+			case ExceptionFormatValueType::FORMAT_VALUE_TYPE_DOUBLE:
+				format_args.push_back(duckdb_fmt::internal::make_arg<duckdb_fmt::printf_context>(val.dbl_val));
+				break;
+			case ExceptionFormatValueType::FORMAT_VALUE_TYPE_INTEGER:
+				format_args.push_back(duckdb_fmt::internal::make_arg<duckdb_fmt::printf_context>(val.int_val));
+				break;
+			case ExceptionFormatValueType::FORMAT_VALUE_TYPE_STRING:
+				format_args.push_back(duckdb_fmt::internal::make_arg<duckdb_fmt::printf_context>(val.str_val));
+				break;
+			}
 		}
-	}
-	return duckdb_fmt::vsprintf(msg, duckdb_fmt::basic_format_args<duckdb_fmt::printf_context>(
-	                                     format_args.data(), static_cast<int>(format_args.size())));
+		return duckdb_fmt::vsprintf(msg, duckdb_fmt::basic_format_args<duckdb_fmt::printf_context>(
+		                                     format_args.data(), static_cast<int>(format_args.size())));
+	} catch (std::exception &ex) { // LCOV_EXCL_START
+		// work-around for oss-fuzz limiting memory which causes issues here
+		if (StringUtil::Contains(ex.what(), "fuzz mode")) {
+			throw Exception(msg);
+		}
+		throw InternalException(std::string("Primary exception: ") + msg +
+		                        "\nSecondary exception in ExceptionFormatValue: " + ex.what());
+	} // LCOV_EXCL_STOP
 }
 
 } // namespace duckdb

package/src/duckdb/src/common/multi_file_reader.cpp

@@ -34,6 +34,9 @@ vector<string> MultiFileReader::GetFileList(ClientContext &context, const Value
 			if (val.IsNull()) {
 				throw ParserException("%s reader cannot take NULL input as parameter", name);
 			}
+			if (val.type().id() != LogicalTypeId::VARCHAR) {
+				throw ParserException("%s reader can only take a list of strings as a parameter", name);
+			}
 			auto glob_files = fs.GlobFiles(StringValue::Get(val), context, options);
 			files.insert(files.end(), glob_files.begin(), glob_files.end());
 		}

package/src/duckdb/src/common/random_engine.cpp

@@ -28,7 +28,7 @@ double RandomEngine::NextRandom(double min, double max) {
 }
 
 double RandomEngine::NextRandom() {
-	return random_state->pcg() / double(std::numeric_limits<uint32_t>::max());
+	return std::ldexp(random_state->pcg(), -32);
 }
 uint32_t RandomEngine::NextRandomInteger() {
 	return random_state->pcg();

package/src/duckdb/src/common/types/row/row_data_collection_scanner.cpp

@@ -101,16 +101,17 @@ void RowDataCollectionScanner::AlignHeapBlocks(RowDataCollection &swizzled_block
 			    make_uniq<RowDataBlock>(buffer_manager, MaxValue<idx_t>(total_size, (idx_t)Storage::BLOCK_SIZE), 1));
 			auto new_heap_handle = buffer_manager.Pin(swizzled_string_heap.blocks.back()->block);
 			auto new_heap_ptr = new_heap_handle.Ptr();
+			for (auto &ptr_and_size : ptrs_and_sizes) {
+				memcpy(new_heap_ptr, ptr_and_size.first, ptr_and_size.second);
+				new_heap_ptr += ptr_and_size.second;
+			}
+			new_heap_ptr = new_heap_handle.Ptr();
 			if (swizzled_string_heap.keep_pinned) {
 				// Since the heap blocks are pinned, we can unswizzle the data again.
 				swizzled_string_heap.pinned_blocks.emplace_back(std::move(new_heap_handle));
 				RowOperations::UnswizzlePointers(layout, base_row_ptr, new_heap_ptr, data_block->count);
 				RowOperations::UnswizzleHeapPointer(layout, base_row_ptr, new_heap_ptr, data_block->count);
 			}
-			for (auto &ptr_and_size : ptrs_and_sizes) {
-				memcpy(new_heap_ptr, ptr_and_size.first, ptr_and_size.second);
-				new_heap_ptr += ptr_and_size.second;
-			}
 		}
 	}
 

package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp

@@ -62,8 +62,21 @@ inline string_t TupleDataWithinListValueLoad(const data_ptr_t &location, data_pt
 	return result;
 }
 
+#ifdef DEBUG
+static void ResetCombinedListData(vector<TupleDataVectorFormat> &vector_data) {
+	for (auto &vd : vector_data) {
+		vd.combined_list_data = nullptr;
+		ResetCombinedListData(vd.child_formats);
+	}
+}
+#endif
+
 void TupleDataCollection::ComputeHeapSizes(TupleDataChunkState &chunk_state, const DataChunk &new_chunk,
                                            const SelectionVector &append_sel, const idx_t append_count) {
+#ifdef DEBUG
+	ResetCombinedListData(chunk_state.vector_data);
+#endif
+
 	auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
 	std::fill_n(heap_sizes, new_chunk.size(), 0);
 
@@ -293,7 +306,12 @@ static void ApplySliceRecursive(const Vector &source_v, TupleDataVectorFormat &s
 		for (idx_t struct_col_idx = 0; struct_col_idx < struct_sources.size(); struct_col_idx++) {
 			auto &struct_source = *struct_sources[struct_col_idx];
 			auto &struct_format = source_format.child_formats[struct_col_idx];
-			struct_format.combined_list_data = make_uniq<CombinedListData>();
+#ifdef DEBUG
+			D_ASSERT(!struct_format.combined_list_data);
+#endif
+			if (!struct_format.combined_list_data) {
+				struct_format.combined_list_data = make_uniq<CombinedListData>();
+			}
 			ApplySliceRecursive(struct_source, struct_format, *source_format.data.sel, count);
 		}
 	}
@@ -308,14 +326,15 @@ void TupleDataCollection::ListWithinListComputeHeapSizes(Vector &heap_sizes_v, c
 	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
-	// Child list
+	// Child list ("source_v")
 	const auto &child_list_data = source_format.data;
 	const auto child_list_sel = *child_list_data.sel;
 	const auto child_list_entries = UnifiedVectorFormat::GetData<list_entry_t>(child_list_data);
 	const auto &child_list_validity = child_list_data.validity;
 
-	// Figure out actual child list size (differs from ListVector::GetListSize if dict/const vector)
-	idx_t child_list_child_count = ListVector::GetListSize(source_v);
+	// Figure out actual child list size (can differ from ListVector::GetListSize if dict/const vector),
+	// and we cannot use ConstantVector::ZeroSelectionVector because it may need to be longer than STANDARD_VECTOR_SIZE
+	idx_t sum_of_sizes = 0;
 	for (idx_t i = 0; i < append_count; i++) {
 		const auto list_idx = list_sel.get_index(append_sel.get_index(i));
 		if (!list_validity.RowIsValid(list_idx)) {
@@ -324,15 +343,33 @@ void TupleDataCollection::ListWithinListComputeHeapSizes(Vector &heap_sizes_v, c
 		const auto &list_entry = list_entries[list_idx];
 		const auto &list_offset = list_entry.offset;
 		const auto &list_length = list_entry.length;
-		child_list_child_count = MaxValue<idx_t>(child_list_child_count, list_offset + list_length);
+
+		for (idx_t child_i = 0; child_i < list_length; child_i++) {
+			const auto child_list_idx = child_list_sel.get_index(list_offset + child_i);
+			if (!child_list_validity.RowIsValid(child_list_idx)) {
+				continue;
+			}
+
+			const auto &child_list_entry = child_list_entries[child_list_idx];
+			const auto &child_list_length = child_list_entry.length;
+
+			sum_of_sizes += child_list_length;
+		}
 	}
+	const auto child_list_child_count = MaxValue<idx_t>(sum_of_sizes, ListVector::GetListSize(source_v));
 
 	// Target
 	auto heap_sizes = FlatVector::GetData<idx_t>(heap_sizes_v);
 
 	// Construct combined list entries and a selection vector for the child list child
 	auto &child_format = source_format.child_formats[0];
-	child_format.combined_list_data = make_uniq<CombinedListData>();
+#ifdef DEBUG
+	// In debug mode this should be deleted by ResetCombinedListData
+	D_ASSERT(!child_format.combined_list_data);
+#endif
+	if (!child_format.combined_list_data) {
+		child_format.combined_list_data = make_uniq<CombinedListData>();
+	}
 	auto &combined_list_data = *child_format.combined_list_data;
 	auto &combined_list_entries = combined_list_data.combined_list_entries;
 	SelectionVector combined_sel(child_list_child_count);
@@ -365,7 +402,7 @@ void TupleDataCollection::ListWithinListComputeHeapSizes(Vector &heap_sizes_v, c
 				const auto &child_list_offset = child_list_entry.offset;
 				const auto &child_list_length = child_list_entry.length;
 
-				// Add this child's list entry's to the combined selection vector
+				// Add this child's list entries to the combined selection vector
 				for (idx_t child_value_i = 0; child_value_i < child_list_length; child_value_i++) {
 					auto idx = combined_list_offset + child_list_size + child_value_i;
 					auto loc = child_list_offset + child_value_i;