duckdb 0.7.2-dev225.0 → 0.7.2-dev2320.0

package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp

@@ -65,8 +65,8 @@ void BufferedCSVReaderOptions::SetHeader(bool input) {
 	this->has_header = true;
 }
 
-void BufferedCSVReaderOptions::SetCompression(const string &compression) {
-	this->compression = FileCompressionTypeFromString(compression);
+void BufferedCSVReaderOptions::SetCompression(const string &compression_p) {
+	this->compression = FileCompressionTypeFromString(compression_p);
 }
 
 void BufferedCSVReaderOptions::SetEscape(const string &input) {
@@ -74,11 +74,6 @@ void BufferedCSVReaderOptions::SetEscape(const string &input) {
 	this->has_escape = true;
 }
 
-void BufferedCSVReaderOptions::SetParallel(bool use_parallel) {
-	this->has_parallel = true;
-	this->use_parallel = use_parallel;
-}
-
 void BufferedCSVReaderOptions::SetDelimiter(const string &input) {
 	this->delimiter = StringUtil::Replace(input, "\\t", "\t");
 	this->has_delimiter = true;
@@ -87,8 +82,8 @@ void BufferedCSVReaderOptions::SetDelimiter(const string &input) {
 	}
 }
 
-void BufferedCSVReaderOptions::SetQuote(const string &quote) {
-	this->quote = quote;
+void BufferedCSVReaderOptions::SetQuote(const string &quote_p) {
+	this->quote = quote_p;
 	this->has_quote = true;
 }
 
@@ -106,12 +101,10 @@ void BufferedCSVReaderOptions::SetNewline(const string &input) {
 void BufferedCSVReaderOptions::SetDateFormat(LogicalTypeId type, const string &format, bool read_format) {
 	string error;
 	if (read_format) {
-		auto &date_format = this->date_format[type];
-		error = StrTimeFormat::ParseFormatSpecifier(format, date_format);
-		date_format.format_specifier = format;
+		error = StrTimeFormat::ParseFormatSpecifier(format, date_format[type]);
+		date_format[type].format_specifier = format;
 	} else {
-		auto &date_format = this->write_date_format[type];
-		error = StrTimeFormat::ParseFormatSpecifier(format, date_format);
+		error = StrTimeFormat::ParseFormatSpecifier(format, write_date_format[type]);
 	}
 	if (!error.empty()) {
 		throw InvalidInputException("Could not parse DATEFORMAT: %s", error.c_str());
@@ -126,8 +119,6 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 	}
 	if (loption == "auto_detect") {
 		auto_detect = ParseBoolean(value, loption);
-	} else if (loption == "parallel") {
-		SetParallel(ParseBoolean(value, loption));
 	} else if (loption == "sample_size") {
 		int64_t sample_size = ParseInteger(value, loption);
 		if (sample_size < 1 && sample_size != -1) {
@@ -145,6 +136,7 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 		}
 	} else if (loption == "skip") {
 		skip_rows = ParseInteger(value, loption);
+		skip_rows_set = true;
 	} else if (loption == "max_line_size" || loption == "maximum_line_size") {
 		maximum_line_size = ParseInteger(value, loption);
 	} else if (loption == "sample_chunk_size") {
@@ -171,8 +163,6 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 		SetDateFormat(LogicalTypeId::TIMESTAMP, format, true);
 	} else if (loption == "ignore_errors") {
 		ignore_errors = ParseBoolean(value, loption);
-	} else if (loption == "union_by_name") {
-		union_by_name = ParseBoolean(value, loption);
 	} else if (loption == "buffer_size") {
 		buffer_size = ParseInteger(value, loption);
 		if (buffer_size == 0) {
@@ -183,6 +173,8 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 		if (decimal_separator != "." && decimal_separator != ",") {
 			throw BinderException("Unsupported parameter for DECIMAL_SEPARATOR: should be '.' or ','");
 		}
+	} else if (loption == "null_padding") {
+		null_padding = ParseBoolean(value, loption);
 	} else {
 		throw BinderException("Unrecognized option for CSV reader \"%s\"", loption);
 	}
@@ -194,7 +186,7 @@ void BufferedCSVReaderOptions::SetWriteOption(const string &loption, const Value
 	}
 
 	if (loption == "force_quote") {
-		force_quote = ParseColumnList(value, names, loption);
+		force_quote = ParseColumnList(value, name_list, loption);
 	} else if (loption == "date_format" || loption == "dateformat") {
 		string format = ParseString(value, loption);
 		SetDateFormat(LogicalTypeId::DATE, format, false);
@@ -204,6 +196,7 @@ void BufferedCSVReaderOptions::SetWriteOption(const string &loption, const Value
 			format = "%Y-%m-%dT%H:%M:%S.%fZ";
 		}
 		SetDateFormat(LogicalTypeId::TIMESTAMP, format, false);
+		SetDateFormat(LogicalTypeId::TIMESTAMP_TZ, format, false);
 	} else {
 		throw BinderException("Unrecognized option CSV writer \"%s\"", loption);
 	}

package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp

@@ -25,8 +25,9 @@
 namespace duckdb {
 
 ParallelCSVReader::ParallelCSVReader(ClientContext &context, BufferedCSVReaderOptions options_p,
-                                     unique_ptr<CSVBufferRead> buffer_p, const vector<LogicalType> &requested_types)
-    : BaseCSVReader(context, std::move(options_p), requested_types) {
+                                     unique_ptr<CSVBufferRead> buffer_p, idx_t first_pos_first_buffer_p,
+                                     const vector<LogicalType> &requested_types)
+    : BaseCSVReader(context, std::move(options_p), requested_types), first_pos_first_buffer(first_pos_first_buffer_p) {
 	Initialize(requested_types);
 	SetBufferRead(std::move(buffer_p));
 	if (options.delimiter.size() > 1 || options.escape.size() > 1 || options.quote.size() > 1) {
@@ -40,7 +41,6 @@ ParallelCSVReader::~ParallelCSVReader() {
 void ParallelCSVReader::Initialize(const vector<LogicalType> &requested_types) {
 	return_types = requested_types;
 	InitParseChunk(return_types.size());
-	InitInsertChunkIdx(return_types.size());
 }
 
 bool ParallelCSVReader::NewLineDelimiter(bool carry, bool carry_followed_by_nl, bool first_char) {
@@ -53,9 +53,34 @@ bool ParallelCSVReader::NewLineDelimiter(bool carry, bool carry_followed_by_nl,
 	return (carry && carry_followed_by_nl) || (!carry && first_char);
 }
 
+void ParallelCSVReader::SkipEmptyLines() {
+	idx_t new_pos_buffer = position_buffer;
+	if (parse_chunk.data.size() == 1) {
+		// Empty lines are null data.
+		return;
+	}
+	for (; new_pos_buffer < end_buffer; new_pos_buffer++) {
+		if (StringUtil::CharacterIsNewline((*buffer)[new_pos_buffer])) {
+			bool carrier_return = (*buffer)[new_pos_buffer] == '\r';
+			new_pos_buffer++;
+			if (carrier_return && new_pos_buffer < buffer_size && (*buffer)[new_pos_buffer] == '\n') {
+				position_buffer++;
+			}
+			if (new_pos_buffer > end_buffer) {
+				return;
+			}
+			position_buffer = new_pos_buffer;
+		} else if ((*buffer)[new_pos_buffer] != ' ') {
+			return;
+		}
+	}
+}
+
 bool ParallelCSVReader::SetPosition(DataChunk &insert_chunk) {
 	if (buffer->buffer->IsCSVFileFirstBuffer() && start_buffer == position_buffer &&
-	    start_buffer == buffer->buffer->GetStart()) {
+	    start_buffer == first_pos_first_buffer) {
+		start_buffer = buffer->buffer->GetStart();
+		position_buffer = start_buffer;
 		verification_positions.beginning_of_first_line = position_buffer;
 		verification_positions.end_of_last_line = position_buffer;
 		// First buffer doesn't need any setting
@@ -71,11 +96,23 @@ bool ParallelCSVReader::SetPosition(DataChunk &insert_chunk) {
 					if (position_buffer > end_buffer) {
 						return false;
 					}
+					SkipEmptyLines();
+					if (verification_positions.beginning_of_first_line == 0) {
+						verification_positions.beginning_of_first_line = position_buffer;
+					}
+
+					verification_positions.end_of_last_line = position_buffer;
 					return true;
 				}
 			}
 			return false;
 		}
+		SkipEmptyLines();
+		if (verification_positions.beginning_of_first_line == 0) {
+			verification_positions.beginning_of_first_line = position_buffer;
+		}
+
+		verification_positions.end_of_last_line = position_buffer;
 		return true;
 	}
 
@@ -86,7 +123,7 @@ bool ParallelCSVReader::SetPosition(DataChunk &insert_chunk) {
 	bool successfully_read_first_line = false;
 	while (!successfully_read_first_line) {
 		DataChunk first_line_chunk;
-		first_line_chunk.Initialize(allocator, insert_chunk.GetTypes());
+		first_line_chunk.Initialize(allocator, return_types);
 		for (; position_buffer < end_buffer; position_buffer++) {
 			if (StringUtil::CharacterIsNewline((*buffer)[position_buffer])) {
 				bool carriage_return = (*buffer)[position_buffer] == '\r';
@@ -103,6 +140,11 @@ bool ParallelCSVReader::SetPosition(DataChunk &insert_chunk) {
 				}
 			}
 		}
+		SkipEmptyLines();
+
+		if (position_buffer > buffer_size) {
+			break;
+		}
 
 		if (position_buffer >= end_buffer && !StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1])) {
 			break;
@@ -115,8 +157,19 @@ bool ParallelCSVReader::SetPosition(DataChunk &insert_chunk) {
 		idx_t position_set = position_buffer;
 		start_buffer = position_buffer;
 		// We check if we can add this line
-		successfully_read_first_line = TryParseSimpleCSV(first_line_chunk, error_message, true);
-
+		// disable the projection pushdown while reading the first line
+		// otherwise the first line parsing can be influenced by which columns we are reading
+		auto column_ids = std::move(reader_data.column_ids);
+		auto column_mapping = std::move(reader_data.column_mapping);
+		InitializeProjection();
+		try {
+			successfully_read_first_line = TryParseSimpleCSV(first_line_chunk, error_message, true);
+		} catch (...) {
+			successfully_read_first_line = false;
+		}
+		// restore the projection pushdown
+		reader_data.column_ids = std::move(column_ids);
+		reader_data.column_mapping = std::move(column_mapping);
 		end_buffer = end_buffer_real;
 		start_buffer = position_set;
 		if (position_buffer >= end_buffer) {
@@ -182,27 +235,55 @@ bool ParallelCSVReader::BufferRemainder() {
 	return true;
 }
 
+void VerifyLineLength(idx_t line_size, idx_t max_line_size) {
+	if (line_size > max_line_size) {
+		// FIXME: this should also output the correct estimated linenumber where it broke
+		throw InvalidInputException("Maximum line size of %llu bytes exceeded!", max_line_size);
+	}
+}
+
+bool AllNewLine(string_t value, idx_t column_amount) {
+	auto value_str = value.GetString();
+	if (value_str.empty() && column_amount == 1) {
+		// This is a one column (empty)
+		return false;
+	}
+	for (idx_t i = 0; i < value.GetSize(); i++) {
+		if (!StringUtil::CharacterIsNewline(value_str[i])) {
+			return false;
+		}
+	}
+	return true;
+}
+
 bool ParallelCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message, bool try_add_line) {
 	// used for parsing algorithm
+	if (start_buffer == buffer_size) {
+		// Nothing to read
+		finished = true;
+		return true;
+	}
 	D_ASSERT(end_buffer <= buffer_size);
 	bool finished_chunk = false;
 	idx_t column = 0;
 	idx_t offset = 0;
 	bool has_quotes = false;
+
 	vector<idx_t> escape_positions;
 	if ((start_buffer == buffer->buffer_start || start_buffer == buffer->buffer_end) && !try_add_line) {
 		// First time reading this buffer piece
 		if (!SetPosition(insert_chunk)) {
-			// This means the buffer size does not contain a new line
-			if (position_buffer - start_buffer == options.buffer_size) {
-				error_message = "Line does not fit in one buffer. Increase the buffer size.";
-				return false;
-			}
 			finished = true;
 			return true;
 		}
 	}
-
+	if (position_buffer == buffer_size) {
+		// Nothing to read
+		finished = true;
+		return true;
+	}
+	// Keep track of line size
+	idx_t line_start = position_buffer;
 	// start parsing the first value
 	goto value_start;
 
@@ -234,11 +315,16 @@ normal : {
 		if (c == options.delimiter[0]) {
 			// delimiter: end the value and add it to the chunk
 			goto add_value;
+		} else if (c == options.quote[0] && try_add_line) {
+			return false;
 		} else if (StringUtil::CharacterIsNewline(c)) {
 			// newline: add row
-			if (column > 0 || try_add_line || insert_chunk.data.size() == 1) {
+			if (column > 0 || try_add_line || parse_chunk.data.size() == 1) {
 				goto add_row;
 			}
+			if (column == 0 && position_buffer == start_buffer) {
+				start_buffer++;
+			}
 		}
 	}
 	if (!BufferRemainder()) {
@@ -277,12 +363,15 @@ add_row : {
 		parse_chunk.Reset();
 		return success;
 	} else {
+		VerifyLineLength(position_buffer - line_start, options.maximum_line_size);
+		line_start = position_buffer;
 		finished_chunk = AddRow(insert_chunk, column, error_message);
 	}
 	// increase position by 1 and move start to the new position
 	offset = 0;
 	has_quotes = false;
-	start_buffer = ++position_buffer;
+	position_buffer++;
+	start_buffer = position_buffer;
 	verification_positions.end_of_last_line = position_buffer;
 	if (reached_remainder_state) {
 		goto final_state;
@@ -301,7 +390,10 @@ add_row : {
 			// newline after carriage return: skip
 			// increase position by 1 and move start to the new position
 			start_buffer = ++position_buffer;
+
+			SkipEmptyLines();
 			verification_positions.end_of_last_line = position_buffer;
+			start_buffer = position_buffer;
 			if (reached_remainder_state) {
 				goto final_state;
 			}
@@ -323,6 +415,9 @@ add_row : {
 			error_message = "Wrong NewLine Identifier. Expecting \\r or \\n";
 			return false;
 		}
+		SkipEmptyLines();
+		verification_positions.end_of_last_line = position_buffer;
+		start_buffer = position_buffer;
 		// \n newline, move to value start
 		if (finished_chunk) {
 			goto final_state;
@@ -382,7 +477,8 @@ unquote : {
 		goto add_value;
 	} else if (StringUtil::CharacterIsNewline(c)) {
 		offset = 1;
-		D_ASSERT(column == insert_chunk.ColumnCount() - 1);
+		// FIXME: should this be an assertion?
+		D_ASSERT(try_add_line || (!try_add_line && column == parse_chunk.ColumnCount() - 1));
 		goto add_row;
 	} else if (position_buffer >= end_buffer) {
 		// reached end of buffer
@@ -439,22 +535,27 @@ final_state : {
 	}
 	// If this is the last buffer, we have to read the last value
 	if (buffer->buffer->IsCSVFileLastBuffer() || (buffer->next_buffer && buffer->next_buffer->IsCSVFileLastBuffer())) {
-		if (column > 0 || try_add_line || (insert_chunk.data.size() == 1 && start_buffer != position_buffer)) {
+		if (column > 0 || start_buffer != position_buffer || try_add_line ||
+		    (insert_chunk.data.size() == 1 && start_buffer != position_buffer)) {
 			// remaining values to be added to the chunk
 			auto str_value = buffer->GetValue(start_buffer, position_buffer, offset);
-			AddValue(str_value, column, escape_positions, has_quotes);
-			if (try_add_line) {
-				bool success = column == return_types.size();
-				if (success) {
+			if (!AllNewLine(str_value, insert_chunk.data.size()) || offset == 0) {
+				AddValue(str_value, column, escape_positions, has_quotes);
+				if (try_add_line) {
+					bool success = column == return_types.size();
+					if (success) {
+						AddRow(insert_chunk, column, error_message);
+						success = Flush(insert_chunk);
+					}
+					parse_chunk.Reset();
+					reached_remainder_state = false;
+					return success;
+				} else {
+					VerifyLineLength(position_buffer - line_start, options.maximum_line_size);
+					line_start = position_buffer;
 					AddRow(insert_chunk, column, error_message);
-					success = Flush(insert_chunk);
+					verification_positions.end_of_last_line = position_buffer;
 				}
-				parse_chunk.Reset();
-				reached_remainder_state = false;
-				return success;
-			} else {
-				AddRow(insert_chunk, column, error_message);
-				verification_positions.end_of_last_line = position_buffer;
 			}
 		}
 	}
@@ -462,11 +563,14 @@ final_state : {
 	if (mode == ParserMode::PARSING) {
 		Flush(insert_chunk);
 	}
-	if (position_buffer != verification_positions.end_of_last_line &&
-	    !StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1])) {
+	if (position_buffer - verification_positions.end_of_last_line > options.buffer_size) {
 		error_message = "Line does not fit in one buffer. Increase the buffer size.";
 		return false;
 	}
+	end_buffer = buffer_size;
+	SkipEmptyLines();
+	end_buffer = buffer->buffer_end;
+	verification_positions.end_of_last_line = position_buffer;
 	if (position_buffer >= end_buffer) {
 		if (position_buffer >= end_buffer) {
 			if (position_buffer == end_buffer && StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1]) &&

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -1,27 +1,28 @@
 #include "duckdb/execution/operator/persistent/physical_batch_insert.hpp"
 
 #include "duckdb/parallel/thread_context.hpp"
-#include "duckdb/parser/parsed_data/create_table_info.hpp"
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/storage/table/row_group_collection.hpp"
 #include "duckdb/storage/table_io_manager.hpp"
 #include "duckdb/transaction/local_storage.hpp"
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
-PhysicalBatchInsert::PhysicalBatchInsert(vector<LogicalType> types, TableCatalogEntry *table,
+PhysicalBatchInsert::PhysicalBatchInsert(vector<LogicalType> types, TableCatalogEntry &table,
                                          physical_index_vector_t<idx_t> column_index_map,
                                          vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality)
     : PhysicalOperator(PhysicalOperatorType::BATCH_INSERT, std::move(types), estimated_cardinality),
-      column_index_map(std::move(column_index_map)), insert_table(table), insert_types(table->GetTypes()),
+      column_index_map(std::move(column_index_map)), insert_table(&table), insert_types(table.GetTypes()),
       bound_defaults(std::move(bound_defaults)) {
 }
 
-PhysicalBatchInsert::PhysicalBatchInsert(LogicalOperator &op, SchemaCatalogEntry *schema,
+PhysicalBatchInsert::PhysicalBatchInsert(LogicalOperator &op, SchemaCatalogEntry &schema,
                                          unique_ptr<BoundCreateTableInfo> info_p, idx_t estimated_cardinality)
     : PhysicalOperator(PhysicalOperatorType::BATCH_CREATE_TABLE_AS, op.types, estimated_cardinality),
-      insert_table(nullptr), schema(schema), info(std::move(info_p)) {
+      insert_table(nullptr), schema(&schema), info(std::move(info_p)) {
 	PhysicalInsert::GetInsertInfo(*info, insert_types, bound_defaults);
 }
 
@@ -95,11 +96,11 @@ public:
 
 class BatchInsertGlobalState : public GlobalSinkState {
 public:
-	explicit BatchInsertGlobalState() : insert_count(0) {
+	explicit BatchInsertGlobalState(DuckTableEntry &table) : table(table), insert_count(0) {
 	}
 
 	mutex lock;
-	DuckTableEntry *table;
+	DuckTableEntry &table;
 	idx_t insert_count;
 	map<idx_t, unique_ptr<RowGroupCollection>> collections;
 
@@ -155,7 +156,8 @@ public:
 	}
 
 	void AddCollection(ClientContext &context, idx_t batch_index, unique_ptr<RowGroupCollection> current_collection,
-	                   OptimisticDataWriter *writer = nullptr, bool *written_to_disk = nullptr) {
+	                   optional_ptr<OptimisticDataWriter> writer = nullptr,
+	                   optional_ptr<bool> written_to_disk = nullptr) {
 		vector<unique_ptr<RowGroupCollection>> merge_collections;
 		idx_t merge_count;
 		{
@@ -191,6 +193,7 @@ public:
 					for (idx_t i = start_batch_index; i <= end_batch_index; i++) {
 						if (i == batch_index) {
 							merge_collections.push_back(std::move(current_collection));
+							current_collection.reset();
 							continue;
 						}
 						auto can_merge = CheckMerge(i, merge_collections);
@@ -203,6 +206,7 @@ public:
 			if (merge_collections.empty()) {
 				// no collections to merge together - add the collection to the batch index
 				collections[batch_index] = std::move(current_collection);
+				current_collection.reset();
 			}
 		}
 		if (!merge_collections.empty()) {
@@ -237,7 +241,7 @@ public:
 	idx_t current_index;
 	TableAppendState current_append_state;
 	unique_ptr<RowGroupCollection> current_collection;
-	OptimisticDataWriter *writer;
+	optional_ptr<OptimisticDataWriter> writer;
 	bool written_to_disk;
 
 	void FlushToDisk() {
@@ -250,10 +254,10 @@ public:
 		writer->FlushToDisk(*current_collection, true);
 	}
 
-	void CreateNewCollection(DuckTableEntry *table, const vector<LogicalType> &insert_types) {
-		auto &table_info = table->GetStorage().info;
-		auto &block_manager = TableIOManager::Get(table->GetStorage()).GetBlockManagerForRowData();
-		current_collection = make_unique<RowGroupCollection>(table_info, block_manager, insert_types, MAX_ROW_ID);
+	void CreateNewCollection(DuckTableEntry &table, const vector<LogicalType> &insert_types) {
+		auto &table_info = table.GetStorage().info;
+		auto &block_manager = TableIOManager::Get(table.GetStorage()).GetBlockManagerForRowData();
+		current_collection = make_uniq<RowGroupCollection>(table_info, block_manager, insert_types, MAX_ROW_ID);
 		current_collection->InitializeEmpty();
 		current_collection->InitializeAppend(current_append_state);
 		written_to_disk = false;
@@ -261,38 +265,39 @@ public:
 };
 
 unique_ptr<GlobalSinkState> PhysicalBatchInsert::GetGlobalSinkState(ClientContext &context) const {
-	auto result = make_unique<BatchInsertGlobalState>();
+	optional_ptr<TableCatalogEntry> table;
 	if (info) {
 		// CREATE TABLE AS
 		D_ASSERT(!insert_table);
 		auto &catalog = *schema->catalog;
-		result->table =
-		    (DuckTableEntry *)catalog.CreateTable(catalog.GetCatalogTransaction(context), schema, info.get());
+		table = (TableCatalogEntry *)catalog.CreateTable(catalog.GetCatalogTransaction(context), *schema.get_mutable(),
+		                                                 info.get());
 	} else {
 		D_ASSERT(insert_table);
 		D_ASSERT(insert_table->IsDuckTable());
-		result->table = (DuckTableEntry *)insert_table;
+		table = insert_table.get_mutable();
 	}
+	auto result = make_uniq<BatchInsertGlobalState>(table->Cast<DuckTableEntry>());
 	return std::move(result);
 }
 
 unique_ptr<LocalSinkState> PhysicalBatchInsert::GetLocalSinkState(ExecutionContext &context) const {
-	return make_unique<BatchInsertLocalState>(context.client, insert_types, bound_defaults);
+	return make_uniq<BatchInsertLocalState>(context.client, insert_types, bound_defaults);
 }
 
 SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p,
                                          DataChunk &chunk) const {
-	auto &gstate = (BatchInsertGlobalState &)state;
-	auto &lstate = (BatchInsertLocalState &)lstate_p;
+	auto &gstate = state.Cast<BatchInsertGlobalState>();
+	auto &lstate = lstate_p.Cast<BatchInsertLocalState>();
 
-	auto table = gstate.table;
+	auto &table = gstate.table;
 	PhysicalInsert::ResolveDefaults(table, chunk, column_index_map, lstate.default_executor, lstate.insert_chunk);
 
 	if (!lstate.current_collection) {
 		lock_guard<mutex> l(gstate.lock);
 		// no collection yet: create a new one
 		lstate.CreateNewCollection(table, insert_types);
-		lstate.writer = gstate.table->GetStorage().CreateOptimisticWriter(context.client);
+		lstate.writer = &table.GetStorage().CreateOptimisticWriter(context.client);
 	} else if (lstate.current_index != lstate.batch_index) {
 		// batch index has changed: move the old collection to the global state and create a new collection
 		TransactionData tdata(0, 0);
@@ -304,7 +309,7 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, GlobalSinkSt
 	}
 	lstate.current_index = lstate.batch_index;
 
-	table->GetStorage().VerifyAppendConstraints(*table, context.client, lstate.insert_chunk);
+	table.GetStorage().VerifyAppendConstraints(table, context.client, lstate.insert_chunk);
 
 	auto new_row_group = lstate.current_collection->Append(lstate.insert_chunk, lstate.current_append_state);
 	if (new_row_group) {
@@ -316,10 +321,10 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, GlobalSinkSt
 
 void PhysicalBatchInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
                                   LocalSinkState &lstate_p) const {
-	auto &gstate = (BatchInsertGlobalState &)gstate_p;
-	auto &lstate = (BatchInsertLocalState &)lstate_p;
+	auto &gstate = gstate_p.Cast<BatchInsertGlobalState>();
+	auto &lstate = lstate_p.Cast<BatchInsertLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
-	context.thread.profiler.Flush(this, &lstate.default_executor, "default_executor", 1);
+	context.thread.profiler.Flush(*this, lstate.default_executor, "default_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
 
 	if (!lstate.current_collection) {
@@ -335,19 +340,19 @@ void PhysicalBatchInsert::Combine(ExecutionContext &context, GlobalSinkState &gs
 
 SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
                                                GlobalSinkState &gstate_p) const {
-	auto &gstate = (BatchInsertGlobalState &)gstate_p;
+	auto &gstate = gstate_p.Cast<BatchInsertGlobalState>();
 
 	// in the finalize, do a final pass over all of the collections we created and try to merge smaller collections
 	// together
 	vector<unique_ptr<CollectionMerger>> mergers;
 	unique_ptr<CollectionMerger> current_merger;
 
-	auto &storage = gstate.table->GetStorage();
+	auto &storage = gstate.table.GetStorage();
 	for (auto &collection : gstate.collections) {
 		if (collection.second->GetTotalRows() < LocalStorage::MERGE_THRESHOLD) {
 			// this collection has very few rows: add it to the merge set
 			if (!current_merger) {
-				current_merger = make_unique<CollectionMerger>(context);
+				current_merger = make_uniq<CollectionMerger>(context);
 			}
 			current_merger->AddCollection(std::move(collection.second));
 		} else {
@@ -358,7 +363,7 @@ SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event,
 				mergers.push_back(std::move(current_merger));
 				current_merger.reset();
 			}
-			auto larger_merger = make_unique<CollectionMerger>(context);
+			auto larger_merger = make_uniq<CollectionMerger>(context);
 			larger_merger->AddCollection(std::move(collection.second));
 			mergers.push_back(std::move(larger_merger));
 		}
@@ -370,11 +375,11 @@ SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event,
 	// now that we have created all of the mergers, perform the actual merging
 	vector<unique_ptr<RowGroupCollection>> final_collections;
 	final_collections.reserve(mergers.size());
-	auto writer = storage.CreateOptimisticWriter(context);
+	auto &writer = storage.CreateOptimisticWriter(context);
 	for (auto &merger : mergers) {
-		final_collections.push_back(merger->Flush(*writer));
+		final_collections.push_back(merger->Flush(writer));
 	}
-	writer->FinalFlush();
+	writer.FinalFlush();
 
 	// finally, merge the row groups into the local storage
 	for (auto &collection : final_collections) {
@@ -395,13 +400,13 @@ public:
 };
 
 unique_ptr<GlobalSourceState> PhysicalBatchInsert::GetGlobalSourceState(ClientContext &context) const {
-	return make_unique<BatchInsertSourceState>();
+	return make_uniq<BatchInsertSourceState>();
 }
 
 void PhysicalBatchInsert::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
                                   LocalSourceState &lstate) const {
-	auto &state = (BatchInsertSourceState &)gstate;
-	auto &insert_gstate = (BatchInsertGlobalState &)*sink_state;
+	auto &state = gstate.Cast<BatchInsertSourceState>();
+	auto &insert_gstate = sink_state->Cast<BatchInsertGlobalState>();
 	if (state.finished) {
 		return;
 	}