duckdb 0.8.2-dev2509.0 → 0.8.2-dev2673.0

package/src/duckdb/src/function/table/arrow.cpp

@@ -9,41 +9,37 @@
 #include "duckdb/function/table/arrow.hpp"
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/parser/parsed_data/create_table_function_info.hpp"
+#include "duckdb/function/table/arrow/arrow_duck_schema.hpp"
 #include "utf8proc_wrapper.hpp"
 
 namespace duckdb {
 
-LogicalType ArrowTableFunction::GetArrowLogicalType(
-    ArrowSchema &schema, std::unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data, idx_t col_idx) {
+unique_ptr<ArrowType> ArrowTableFunction::GetArrowLogicalType(ArrowSchema &schema) {
 	auto format = string(schema.format);
-	if (arrow_convert_data.find(col_idx) == arrow_convert_data.end()) {
-		arrow_convert_data[col_idx] = make_uniq<ArrowConvertData>();
-	}
-	auto &convert_data = *arrow_convert_data[col_idx];
 	if (format == "n") {
-		return LogicalType::SQLNULL;
+		return make_uniq<ArrowType>(LogicalType::SQLNULL);
 	} else if (format == "b") {
-		return LogicalType::BOOLEAN;
+		return make_uniq<ArrowType>(LogicalType::BOOLEAN);
 	} else if (format == "c") {
-		return LogicalType::TINYINT;
+		return make_uniq<ArrowType>(LogicalType::TINYINT);
 	} else if (format == "s") {
-		return LogicalType::SMALLINT;
+		return make_uniq<ArrowType>(LogicalType::SMALLINT);
 	} else if (format == "i") {
-		return LogicalType::INTEGER;
+		return make_uniq<ArrowType>(LogicalType::INTEGER);
 	} else if (format == "l") {
-		return LogicalType::BIGINT;
+		return make_uniq<ArrowType>(LogicalType::BIGINT);
 	} else if (format == "C") {
-		return LogicalType::UTINYINT;
+		return make_uniq<ArrowType>(LogicalType::UTINYINT);
 	} else if (format == "S") {
-		return LogicalType::USMALLINT;
+		return make_uniq<ArrowType>(LogicalType::USMALLINT);
 	} else if (format == "I") {
-		return LogicalType::UINTEGER;
+		return make_uniq<ArrowType>(LogicalType::UINTEGER);
 	} else if (format == "L") {
-		return LogicalType::UBIGINT;
+		return make_uniq<ArrowType>(LogicalType::UBIGINT);
 	} else if (format == "f") {
-		return LogicalType::FLOAT;
+		return make_uniq<ArrowType>(LogicalType::FLOAT);
 	} else if (format == "g") {
-		return LogicalType::DOUBLE;
+		return make_uniq<ArrowType>(LogicalType::DOUBLE);
 	} else if (format[0] == 'd') { //! this can be either decimal128 or decimal 256 (e.g., d:38,0)
 		std::string parameters = format.substr(format.find(':'));
 		uint8_t width = std::stoi(parameters.substr(1, parameters.find(',')));
@@ -51,82 +47,74 @@ LogicalType ArrowTableFunction::GetArrowLogicalType(
 		if (width > 38) {
 			throw NotImplementedException("Unsupported Internal Arrow Type for Decimal %s", format);
 		}
-		return LogicalType::DECIMAL(width, scale);
+		return make_uniq<ArrowType>(LogicalType::DECIMAL(width, scale));
 	} else if (format == "u") {
-		convert_data.variable_sz_type.emplace_back(ArrowVariableSizeType::NORMAL, 0);
-		return LogicalType::VARCHAR;
+		return make_uniq<ArrowType>(LogicalType::VARCHAR, ArrowVariableSizeType::NORMAL);
 	} else if (format == "U") {
-		convert_data.variable_sz_type.emplace_back(ArrowVariableSizeType::SUPER_SIZE, 0);
-		return LogicalType::VARCHAR;
+		return make_uniq<ArrowType>(LogicalType::VARCHAR, ArrowVariableSizeType::SUPER_SIZE);
 	} else if (format == "tsn:") {
-		return LogicalTypeId::TIMESTAMP_NS;
+		return make_uniq<ArrowType>(LogicalTypeId::TIMESTAMP_NS);
 	} else if (format == "tsu:") {
-		return LogicalTypeId::TIMESTAMP;
+		return make_uniq<ArrowType>(LogicalTypeId::TIMESTAMP);
 	} else if (format == "tsm:") {
-		return LogicalTypeId::TIMESTAMP_MS;
+		return make_uniq<ArrowType>(LogicalTypeId::TIMESTAMP_MS);
 	} else if (format == "tss:") {
-		return LogicalTypeId::TIMESTAMP_SEC;
+		return make_uniq<ArrowType>(LogicalTypeId::TIMESTAMP_SEC);
 	} else if (format == "tdD") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::DAYS);
-		return LogicalType::DATE;
+		return make_uniq<ArrowType>(LogicalType::DATE, ArrowDateTimeType::DAYS);
 	} else if (format == "tdm") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::MILLISECONDS);
-		return LogicalType::DATE;
+		return make_uniq<ArrowType>(LogicalType::DATE, ArrowDateTimeType::MILLISECONDS);
 	} else if (format == "tts") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::SECONDS);
-		return LogicalType::TIME;
+		return make_uniq<ArrowType>(LogicalType::TIME, ArrowDateTimeType::SECONDS);
 	} else if (format == "ttm") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::MILLISECONDS);
-		return LogicalType::TIME;
+		return make_uniq<ArrowType>(LogicalType::TIME, ArrowDateTimeType::MILLISECONDS);
 	} else if (format == "ttu") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::MICROSECONDS);
-		return LogicalType::TIME;
+		return make_uniq<ArrowType>(LogicalType::TIME, ArrowDateTimeType::MICROSECONDS);
 	} else if (format == "ttn") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::NANOSECONDS);
-		return LogicalType::TIME;
+		return make_uniq<ArrowType>(LogicalType::TIME, ArrowDateTimeType::NANOSECONDS);
 	} else if (format == "tDs") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::SECONDS);
-		return LogicalType::INTERVAL;
+		return make_uniq<ArrowType>(LogicalType::INTERVAL, ArrowDateTimeType::SECONDS);
 	} else if (format == "tDm") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::MILLISECONDS);
-		return LogicalType::INTERVAL;
+		return make_uniq<ArrowType>(LogicalType::INTERVAL, ArrowDateTimeType::MILLISECONDS);
 	} else if (format == "tDu") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::MICROSECONDS);
-		return LogicalType::INTERVAL;
+		return make_uniq<ArrowType>(LogicalType::INTERVAL, ArrowDateTimeType::MICROSECONDS);
 	} else if (format == "tDn") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::NANOSECONDS);
-		return LogicalType::INTERVAL;
+		return make_uniq<ArrowType>(LogicalType::INTERVAL, ArrowDateTimeType::NANOSECONDS);
 	} else if (format == "tiD") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::DAYS);
-		return LogicalType::INTERVAL;
+		return make_uniq<ArrowType>(LogicalType::INTERVAL, ArrowDateTimeType::DAYS);
 	} else if (format == "tiM") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::MONTHS);
-		return LogicalType::INTERVAL;
+		return make_uniq<ArrowType>(LogicalType::INTERVAL, ArrowDateTimeType::MONTHS);
 	} else if (format == "tin") {
-		convert_data.date_time_precision.emplace_back(ArrowDateTimeType::MONTH_DAY_NANO);
-		return LogicalType::INTERVAL;
+		return make_uniq<ArrowType>(LogicalType::INTERVAL, ArrowDateTimeType::MONTH_DAY_NANO);
 	} else if (format == "+l") {
-		convert_data.variable_sz_type.emplace_back(ArrowVariableSizeType::NORMAL, 0);
-		auto child_type = GetArrowLogicalType(*schema.children[0], arrow_convert_data, col_idx);
-		return LogicalType::LIST(child_type);
+		auto child_type = GetArrowLogicalType(*schema.children[0]);
+		auto list_type =
+		    make_uniq<ArrowType>(LogicalType::LIST(child_type->GetDuckType()), ArrowVariableSizeType::NORMAL);
+		list_type->AddChild(std::move(child_type));
+		return list_type;
 	} else if (format == "+L") {
-		convert_data.variable_sz_type.emplace_back(ArrowVariableSizeType::SUPER_SIZE, 0);
-		auto child_type = GetArrowLogicalType(*schema.children[0], arrow_convert_data, col_idx);
-		return LogicalType::LIST(child_type);
+		auto child_type = GetArrowLogicalType(*schema.children[0]);
+		auto list_type =
+		    make_uniq<ArrowType>(LogicalType::LIST(child_type->GetDuckType()), ArrowVariableSizeType::SUPER_SIZE);
+		list_type->AddChild(std::move(child_type));
+		return list_type;
 	} else if (format[0] == '+' && format[1] == 'w') {
 		std::string parameters = format.substr(format.find(':') + 1);
 		idx_t fixed_size = std::stoi(parameters);
-		convert_data.variable_sz_type.emplace_back(ArrowVariableSizeType::FIXED_SIZE, fixed_size);
-		auto child_type = GetArrowLogicalType(*schema.children[0], arrow_convert_data, col_idx);
-		return LogicalType::LIST(child_type);
+		auto child_type = GetArrowLogicalType(*schema.children[0]);
+		auto list_type = make_uniq<ArrowType>(LogicalType::LIST(child_type->GetDuckType()), fixed_size);
+		list_type->AddChild(std::move(child_type));
+		return list_type;
 	} else if (format == "+s") {
 		child_list_t<LogicalType> child_types;
+		vector<unique_ptr<ArrowType>> children;
 		for (idx_t type_idx = 0; type_idx < (idx_t)schema.n_children; type_idx++) {
-			auto child_type = GetArrowLogicalType(*schema.children[type_idx], arrow_convert_data, col_idx);
-			child_types.emplace_back(schema.children[type_idx]->name, child_type);
+			children.emplace_back(GetArrowLogicalType(*schema.children[type_idx]));
+			child_types.emplace_back(schema.children[type_idx]->name, children.back()->GetDuckType());
 		}
-		return LogicalType::STRUCT(child_types);
-
+		auto struct_type = make_uniq<ArrowType>(LogicalType::STRUCT(std::move(child_types)));
+		struct_type->AssignChildren(std::move(children));
+		return struct_type;
 	} else if (format[0] == '+' && format[1] == 'u') {
 		if (format[2] != 's') {
 			throw NotImplementedException("Unsupported Internal Arrow Type: \"%c\" Union", format[2]);
@@ -138,48 +126,59 @@ LogicalType ArrowTableFunction::GetArrowLogicalType(
 		auto type_ids = StringUtil::Split(format.substr(prefix.size()), ',');
 
 		child_list_t<LogicalType> members;
+		vector<unique_ptr<ArrowType>> children;
 		for (idx_t type_idx = 0; type_idx < (idx_t)schema.n_children; type_idx++) {
 			auto type = schema.children[type_idx];
 
-			members.emplace_back(type->name, GetArrowLogicalType(*type, arrow_convert_data, col_idx));
+			children.emplace_back(GetArrowLogicalType(*type));
+			members.emplace_back(type->name, children.back()->GetDuckType());
 		}
 
-		return LogicalType::UNION(members);
-
+		auto union_type = make_uniq<ArrowType>(LogicalType::UNION(members));
+		union_type->AssignChildren(std::move(children));
+		return union_type;
 	} else if (format == "+m") {
-		convert_data.variable_sz_type.emplace_back(ArrowVariableSizeType::NORMAL, 0);
-
 		auto &arrow_struct_type = *schema.children[0];
 		D_ASSERT(arrow_struct_type.n_children == 2);
-		auto key_type = GetArrowLogicalType(*arrow_struct_type.children[0], arrow_convert_data, col_idx);
-		auto value_type = GetArrowLogicalType(*arrow_struct_type.children[1], arrow_convert_data, col_idx);
-		return LogicalType::MAP(key_type, value_type);
+		auto key_type = GetArrowLogicalType(*arrow_struct_type.children[0]);
+		auto value_type = GetArrowLogicalType(*arrow_struct_type.children[1]);
+		auto map_type = make_uniq<ArrowType>(LogicalType::MAP(key_type->GetDuckType(), value_type->GetDuckType()),
+		                                     ArrowVariableSizeType::NORMAL);
+		child_list_t<LogicalType> key_value;
+		key_value.emplace_back(std::make_pair("key", key_type->GetDuckType()));
+		key_value.emplace_back(std::make_pair("value", value_type->GetDuckType()));
+
+		auto inner_struct =
+		    make_uniq<ArrowType>(LogicalType::STRUCT(std::move(key_value)), ArrowVariableSizeType::NORMAL);
+		vector<unique_ptr<ArrowType>> children;
+		children.reserve(2);
+		children.push_back(std::move(key_type));
+		children.push_back(std::move(value_type));
+		inner_struct->AssignChildren(std::move(children));
+		map_type->AddChild(std::move(inner_struct));
+		return map_type;
 	} else if (format == "z") {
-		convert_data.variable_sz_type.emplace_back(ArrowVariableSizeType::NORMAL, 0);
-		return LogicalType::BLOB;
+		return make_uniq<ArrowType>(LogicalType::BLOB, ArrowVariableSizeType::NORMAL);
 	} else if (format == "Z") {
-		convert_data.variable_sz_type.emplace_back(ArrowVariableSizeType::SUPER_SIZE, 0);
-		return LogicalType::BLOB;
+		return make_uniq<ArrowType>(LogicalType::BLOB, ArrowVariableSizeType::SUPER_SIZE);
 	} else if (format[0] == 'w') {
 		std::string parameters = format.substr(format.find(':') + 1);
 		idx_t fixed_size = std::stoi(parameters);
-		convert_data.variable_sz_type.emplace_back(ArrowVariableSizeType::FIXED_SIZE, fixed_size);
-		return LogicalType::BLOB;
+		return make_uniq<ArrowType>(LogicalType::BLOB, fixed_size);
 	} else if (format[0] == 't' && format[1] == 's') {
 		// Timestamp with Timezone
+		// TODO right now we just get the UTC value. We probably want to support this properly in the future
 		if (format[2] == 'n') {
-			convert_data.date_time_precision.emplace_back(ArrowDateTimeType::NANOSECONDS);
+			return make_uniq<ArrowType>(LogicalType::TIMESTAMP_TZ, ArrowDateTimeType::NANOSECONDS);
 		} else if (format[2] == 'u') {
-			convert_data.date_time_precision.emplace_back(ArrowDateTimeType::MICROSECONDS);
+			return make_uniq<ArrowType>(LogicalType::TIMESTAMP_TZ, ArrowDateTimeType::MICROSECONDS);
 		} else if (format[2] == 'm') {
-			convert_data.date_time_precision.emplace_back(ArrowDateTimeType::MILLISECONDS);
+			return make_uniq<ArrowType>(LogicalType::TIMESTAMP_TZ, ArrowDateTimeType::MILLISECONDS);
 		} else if (format[2] == 's') {
-			convert_data.date_time_precision.emplace_back(ArrowDateTimeType::SECONDS);
+			return make_uniq<ArrowType>(LogicalType::TIMESTAMP_TZ, ArrowDateTimeType::SECONDS);
 		} else {
 			throw NotImplementedException(" Timestamptz precision of not accepted");
 		}
-		// TODO right now we just get the UTC value. We probably want to support this properly in the future
-		return LogicalType::TIMESTAMP_TZ;
 	} else {
 		throw NotImplementedException("Unsupported Internal Arrow Type %s", format);
 	}
@@ -224,13 +223,17 @@ unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBind(ClientContext &contex
 		if (!schema.release) {
 			throw InvalidInputException("arrow_scan: released schema passed");
 		}
+		auto arrow_type = GetArrowLogicalType(schema);
 		if (schema.dictionary) {
-			auto logical_type = GetArrowLogicalType(schema, res->arrow_convert_data, col_idx);
-			res->arrow_convert_data[col_idx] = make_uniq<ArrowConvertData>(std::move(logical_type));
-			return_types.emplace_back(GetArrowLogicalType(*schema.dictionary, res->arrow_convert_data, col_idx));
+			auto logical_type = arrow_type->GetDuckType();
+			auto dictionary = GetArrowLogicalType(*schema.dictionary);
+			return_types.emplace_back(dictionary->GetDuckType());
+			// The dictionary might have different attributes (size type, datetime precision, etc..)
+			arrow_type->SetDictionary(std::move(dictionary));
 		} else {
-			return_types.emplace_back(GetArrowLogicalType(schema, res->arrow_convert_data, col_idx));
+			return_types.emplace_back(arrow_type->GetDuckType());
 		}
+		res->arrow_table.AddColumn(col_idx, std::move(arrow_type));
 		auto format = string(schema.format);
 		auto name = string(schema.name);
 		if (name.empty()) {
@@ -348,11 +351,11 @@ void ArrowTableFunction::ArrowScanFunction(ClientContext &context, TableFunction
 	if (global_state.CanRemoveFilterColumns()) {
 		state.all_columns.Reset();
 		state.all_columns.SetCardinality(output_size);
-		ArrowToDuckDB(state, data.arrow_convert_data, state.all_columns, data.lines_read - output_size);
+		ArrowToDuckDB(state, data.arrow_table.GetColumns(), state.all_columns, data.lines_read - output_size);
 		output.ReferenceColumns(state.all_columns, global_state.projection_ids);
 	} else {
 		output.SetCardinality(output_size);
-		ArrowToDuckDB(state, data.arrow_convert_data, output, data.lines_read - output_size);
+		ArrowToDuckDB(state, data.arrow_table.GetColumns(), output, data.lines_read - output_size);
 	}
 
 	output.Verify();

package/src/duckdb/src/function/table/arrow_conversion.cpp

@@ -5,16 +5,6 @@
 #include "duckdb/common/types/arrow_aux_data.hpp"
 #include "duckdb/function/scalar/nested_functions.hpp"
 
-namespace {
-using duckdb::idx_t;
-struct ArrowConvertDataIndices {
-	//! The index that refers to 'variable_sz_type' in ArrowConvertData
-	idx_t variable_sized_index;
-	//! The index that refers to 'date_time_precision' in ArrowConvertData
-	idx_t datetime_precision_index;
-};
-} // namespace
-
 namespace duckdb {
 
 static void ShiftRight(unsigned char *ar, int size, int shift) {
@@ -91,35 +81,33 @@ static void SetValidityMask(Vector &vector, ArrowArray &array, ArrowScanLocalSta
 }
 
 static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
-                                std::unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data,
-                                idx_t col_idx, ArrowConvertDataIndices &arrow_convert_idx, int64_t nested_offset = -1,
+                                const ArrowType &arrow_type, int64_t nested_offset = -1,
                                 ValidityMask *parent_mask = nullptr, uint64_t parent_offset = 0);
 
 static void ArrowToDuckDBList(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
-                              std::unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data,
-                              idx_t col_idx, ArrowConvertDataIndices &arrow_convert_idx, int64_t nested_offset,
-                              ValidityMask *parent_mask) {
-	auto original_type = arrow_convert_data[col_idx]->variable_sz_type[arrow_convert_idx.variable_sized_index++];
+                              const ArrowType &arrow_type, int64_t nested_offset, ValidityMask *parent_mask) {
+	auto size_type = arrow_type.GetSizeType();
 	idx_t list_size = 0;
 	SetValidityMask(vector, array, scan_state, size, nested_offset);
 	idx_t start_offset = 0;
 	idx_t cur_offset = 0;
-	if (original_type.first == ArrowVariableSizeType::FIXED_SIZE) {
+	if (size_type == ArrowVariableSizeType::FIXED_SIZE) {
+		auto fixed_size = arrow_type.FixedSize();
 		//! Have to check validity mask before setting this up
-		idx_t offset = (scan_state.chunk_offset + array.offset) * original_type.second;
+		idx_t offset = (scan_state.chunk_offset + array.offset) * fixed_size;
 		if (nested_offset != -1) {
-			offset = original_type.second * nested_offset;
+			offset = fixed_size * nested_offset;
 		}
 		start_offset = offset;
 		auto list_data = FlatVector::GetData<list_entry_t>(vector);
 		for (idx_t i = 0; i < size; i++) {
 			auto &le = list_data[i];
 			le.offset = cur_offset;
-			le.length = original_type.second;
-			cur_offset += original_type.second;
+			le.length = fixed_size;
+			cur_offset += fixed_size;
 		}
 		list_size = start_offset + cur_offset;
-	} else if (original_type.first == ArrowVariableSizeType::NORMAL) {
+	} else if (size_type == ArrowVariableSizeType::NORMAL) {
 		auto offsets = ArrowBufferData<uint32_t>(array, 1) + array.offset + scan_state.chunk_offset;
 		if (nested_offset != -1) {
 			offsets = ArrowBufferData<uint32_t>(array, 1) + nested_offset;
@@ -165,24 +153,22 @@ static void ArrowToDuckDBList(Vector &vector, ArrowArray &array, ArrowScanLocalS
 		}
 	}
 	if (list_size == 0 && start_offset == 0) {
-		ColumnArrowToDuckDB(child_vector, *array.children[0], scan_state, list_size, arrow_convert_data, col_idx,
-		                    arrow_convert_idx, -1);
+		ColumnArrowToDuckDB(child_vector, *array.children[0], scan_state, list_size, arrow_type[0], -1);
 	} else {
-		ColumnArrowToDuckDB(child_vector, *array.children[0], scan_state, list_size, arrow_convert_data, col_idx,
-		                    arrow_convert_idx, start_offset);
+		ColumnArrowToDuckDB(child_vector, *array.children[0], scan_state, list_size, arrow_type[0], start_offset);
 	}
 }
 
 static void ArrowToDuckDBBlob(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
-                              std::unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data,
-                              idx_t col_idx, ArrowConvertDataIndices &arrow_convert_idx, int64_t nested_offset) {
-	auto original_type = arrow_convert_data[col_idx]->variable_sz_type[arrow_convert_idx.variable_sized_index++];
+                              const ArrowType &arrow_type, int64_t nested_offset) {
+	auto size_type = arrow_type.GetSizeType();
 	SetValidityMask(vector, array, scan_state, size, nested_offset);
-	if (original_type.first == ArrowVariableSizeType::FIXED_SIZE) {
+	if (size_type == ArrowVariableSizeType::FIXED_SIZE) {
+		auto fixed_size = arrow_type.FixedSize();
 		//! Have to check validity mask before setting this up
-		idx_t offset = (scan_state.chunk_offset + array.offset) * original_type.second;
+		idx_t offset = (scan_state.chunk_offset + array.offset) * fixed_size;
 		if (nested_offset != -1) {
-			offset = original_type.second * nested_offset;
+			offset = fixed_size * nested_offset;
 		}
 		auto cdata = ArrowBufferData<char>(array, 1);
 		for (idx_t row_idx = 0; row_idx < size; row_idx++) {
@@ -190,11 +176,11 @@ static void ArrowToDuckDBBlob(Vector &vector, ArrowArray &array, ArrowScanLocalS
 				continue;
 			}
 			auto bptr = cdata + offset;
-			auto blob_len = original_type.second;
+			auto blob_len = fixed_size;
 			FlatVector::GetData<string_t>(vector)[row_idx] = StringVector::AddStringOrBlob(vector, bptr, blob_len);
 			offset += blob_len;
 		}
-	} else if (original_type.first == ArrowVariableSizeType::NORMAL) {
+	} else if (size_type == ArrowVariableSizeType::NORMAL) {
 		auto offsets = ArrowBufferData<uint32_t>(array, 1) + array.offset + scan_state.chunk_offset;
 		if (nested_offset != -1) {
 			offsets = ArrowBufferData<uint32_t>(array, 1) + array.offset + nested_offset;
@@ -358,9 +344,8 @@ static void IntervalConversionMonthDayNanos(Vector &vector, ArrowArray &array, A
 }
 
 static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
-                                std::unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data,
-                                idx_t col_idx, ArrowConvertDataIndices &arrow_convert_idx, int64_t nested_offset,
-                                ValidityMask *parent_mask, uint64_t parent_offset) {
+                                const ArrowType &arrow_type, int64_t nested_offset, ValidityMask *parent_mask,
+                                uint64_t parent_offset) {
 	switch (vector.GetType().id()) {
 	case LogicalTypeId::SQLNULL:
 		vector.Reference(Value());
@@ -412,9 +397,9 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
 		break;
 	}
 	case LogicalTypeId::VARCHAR: {
-		auto original_type = arrow_convert_data[col_idx]->variable_sz_type[arrow_convert_idx.variable_sized_index++];
+		auto size_type = arrow_type.GetSizeType();
 		auto cdata = ArrowBufferData<char>(array, 2);
-		if (original_type.first == ArrowVariableSizeType::SUPER_SIZE) {
+		if (size_type == ArrowVariableSizeType::SUPER_SIZE) {
 			auto offsets = ArrowBufferData<uint64_t>(array, 1) + array.offset + scan_state.chunk_offset;
 			if (nested_offset != -1) {
 				offsets = ArrowBufferData<uint64_t>(array, 1) + array.offset + nested_offset;
@@ -430,7 +415,8 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
 		break;
 	}
 	case LogicalTypeId::DATE: {
-		auto precision = arrow_convert_data[col_idx]->date_time_precision[arrow_convert_idx.datetime_precision_index++];
+
+		auto precision = arrow_type.GetDateTimeType();
 		switch (precision) {
 		case ArrowDateTimeType::DAYS: {
 			DirectConversion(vector, array, scan_state, nested_offset, parent_offset);
@@ -454,7 +440,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
 		break;
 	}
 	case LogicalTypeId::TIME: {
-		auto precision = arrow_convert_data[col_idx]->date_time_precision[arrow_convert_idx.datetime_precision_index++];
+		auto precision = arrow_type.GetDateTimeType();
 		switch (precision) {
 		case ArrowDateTimeType::SECONDS: {
 			TimeConversion<int32_t>(vector, array, scan_state, nested_offset, size, 1000000);
@@ -485,7 +471,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
 		break;
 	}
 	case LogicalTypeId::TIMESTAMP_TZ: {
-		auto precision = arrow_convert_data[col_idx]->date_time_precision[arrow_convert_idx.datetime_precision_index++];
+		auto precision = arrow_type.GetDateTimeType();
 		switch (precision) {
 		case ArrowDateTimeType::SECONDS: {
 			TimestampTZConversion(vector, array, scan_state, nested_offset, size, 1000000);
@@ -516,7 +502,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
 		break;
 	}
 	case LogicalTypeId::INTERVAL: {
-		auto precision = arrow_convert_data[col_idx]->date_time_precision[arrow_convert_idx.datetime_precision_index++];
+		auto precision = arrow_type.GetDateTimeType();
 		switch (precision) {
 		case ArrowDateTimeType::SECONDS: {
 			IntervalConversionUs(vector, array, scan_state, nested_offset, size, 1000000);
@@ -611,18 +597,15 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
 		break;
 	}
 	case LogicalTypeId::BLOB: {
-		ArrowToDuckDBBlob(vector, array, scan_state, size, arrow_convert_data, col_idx, arrow_convert_idx,
-		                  nested_offset);
+		ArrowToDuckDBBlob(vector, array, scan_state, size, arrow_type, nested_offset);
 		break;
 	}
 	case LogicalTypeId::LIST: {
-		ArrowToDuckDBList(vector, array, scan_state, size, arrow_convert_data, col_idx, arrow_convert_idx,
-		                  nested_offset, parent_mask);
+		ArrowToDuckDBList(vector, array, scan_state, size, arrow_type, nested_offset, parent_mask);
 		break;
 	}
 	case LogicalTypeId::MAP: {
-		ArrowToDuckDBList(vector, array, scan_state, size, arrow_convert_data, col_idx, arrow_convert_idx,
-		                  nested_offset, parent_mask);
+		ArrowToDuckDBList(vector, array, scan_state, size, arrow_type, nested_offset, parent_mask);
 		ArrowToDuckDBMapVerify(vector, size);
 		break;
 	}
@@ -630,7 +613,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
 		//! Fill the children
 		auto &child_entries = StructVector::GetEntries(vector);
 		auto &struct_validity_mask = FlatVector::Validity(vector);
-		for (idx_t type_idx = 0; type_idx < (idx_t)array.n_children; type_idx++) {
+		for (idx_t type_idx = 0; type_idx < static_cast<idx_t>(array.n_children); type_idx++) {
 			SetValidityMask(*child_entries[type_idx], *array.children[type_idx], scan_state, size, nested_offset);
 			if (!struct_validity_mask.AllValid()) {
 				auto &child_validity_mark = FlatVector::Validity(*child_entries[type_idx]);
@@ -641,8 +624,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
 				}
 			}
 			ColumnArrowToDuckDB(*child_entries[type_idx], *array.children[type_idx], scan_state, size,
-			                    arrow_convert_data, col_idx, arrow_convert_idx, nested_offset, &struct_validity_mask,
-			                    array.offset);
+			                    arrow_type[type_idx], nested_offset, &struct_validity_mask, array.offset);
 		}
 		break;
 	}
@@ -654,14 +636,13 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
 		auto &validity_mask = FlatVector::Validity(vector);
 
 		duckdb::vector<Vector> children;
-		for (idx_t type_idx = 0; type_idx < (::idx_t)array.n_children; type_idx++) {
+		for (idx_t type_idx = 0; type_idx < static_cast<idx_t>(array.n_children); type_idx++) {
 			Vector child(members[type_idx].second);
 			auto arrow_array = array.children[type_idx];
 
 			SetValidityMask(child, *arrow_array, scan_state, size, nested_offset);
 
-			ColumnArrowToDuckDB(child, *arrow_array, scan_state, size, arrow_convert_data, col_idx, arrow_convert_idx,
-			                    nested_offset, &validity_mask);
+			ColumnArrowToDuckDB(child, *arrow_array, scan_state, size, arrow_type, nested_offset, &validity_mask);
 
 			children.push_back(std::move(child));
 		}
@@ -809,20 +790,18 @@ static void SetSelectionVector(SelectionVector &sel, data_ptr_t indices_p, Logic
 }
 
 static void ColumnArrowToDuckDBDictionary(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state,
-                                          idx_t size,
-                                          std::unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data,
-                                          idx_t col_idx, ArrowConvertDataIndices &arrow_convert_idx) {
+                                          idx_t size, const ArrowType &arrow_type, idx_t col_idx) {
 	SelectionVector sel;
 	auto &dict_vectors = scan_state.arrow_dictionary_vectors;
 	if (!dict_vectors.count(col_idx)) {
 		//! We need to set the dictionary data for this column
 		auto base_vector = make_uniq<Vector>(vector.GetType(), array.dictionary->length);
 		SetValidityMask(*base_vector, *array.dictionary, scan_state, array.dictionary->length, 0, array.null_count > 0);
-		ColumnArrowToDuckDB(*base_vector, *array.dictionary, scan_state, array.dictionary->length, arrow_convert_data,
-		                    col_idx, arrow_convert_idx);
+		ColumnArrowToDuckDB(*base_vector, *array.dictionary, scan_state, array.dictionary->length,
+		                    arrow_type.GetDictionary());
 		dict_vectors[col_idx] = std::move(base_vector);
 	}
-	auto dictionary_type = arrow_convert_data[col_idx]->dictionary_type;
+	auto dictionary_type = arrow_type.GetDuckType();
 	//! Get Pointer to Indices of Dictionary
 	auto indices = ArrowBufferData<data_t>(array, 1) +
 	               GetTypeIdSize(dictionary_type.InternalType()) * (scan_state.chunk_offset + array.offset);
@@ -836,8 +815,7 @@ static void ColumnArrowToDuckDBDictionary(Vector &vector, ArrowArray &array, Arr
 	vector.Slice(*dict_vectors[col_idx], sel, size);
 }
 
-void ArrowTableFunction::ArrowToDuckDB(ArrowScanLocalState &scan_state,
-                                       unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data,
+void ArrowTableFunction::ArrowToDuckDB(ArrowScanLocalState &scan_state, const arrow_column_map_t &arrow_convert_data,
                                        DataChunk &output, idx_t start, bool arrow_scan_is_projected) {
 	for (idx_t idx = 0; idx < output.ColumnCount(); idx++) {
 		auto col_idx = scan_state.column_ids[idx];
@@ -851,7 +829,6 @@ void ArrowTableFunction::ArrowToDuckDB(ArrowScanLocalState &scan_state,
 			continue;
 		}
 
-		ArrowConvertDataIndices arrow_convert_idx {0, 0};
 		auto &array = *scan_state.chunk->arrow_array.children[arrow_array_idx];
 		if (!array.release) {
 			throw InvalidInputException("arrow_scan: released array passed");
@@ -861,13 +838,13 @@ void ArrowTableFunction::ArrowToDuckDB(ArrowScanLocalState &scan_state,
 		}
 		// Make sure this Vector keeps the Arrow chunk alive in case we can zero-copy the data
 		output.data[idx].GetBuffer()->SetAuxiliaryData(make_uniq<ArrowAuxiliaryData>(scan_state.chunk));
+		D_ASSERT(arrow_convert_data.find(col_idx) != arrow_convert_data.end());
+		auto &arrow_type = *arrow_convert_data.at(col_idx);
 		if (array.dictionary) {
-			ColumnArrowToDuckDBDictionary(output.data[idx], array, scan_state, output.size(), arrow_convert_data,
-			                              col_idx, arrow_convert_idx);
+			ColumnArrowToDuckDBDictionary(output.data[idx], array, scan_state, output.size(), arrow_type, col_idx);
 		} else {
 			SetValidityMask(output.data[idx], array, scan_state, output.size(), -1);
-			ColumnArrowToDuckDB(output.data[idx], array, scan_state, output.size(), arrow_convert_data, col_idx,
-			                    arrow_convert_idx);
+			ColumnArrowToDuckDB(output.data[idx], array, scan_state, output.size(), arrow_type);
 		}
 	}
 }

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev2509"
+#define DUCKDB_VERSION "0.8.2-dev2673"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "785b11edd5"
+#define DUCKDB_SOURCE_ID "fd683b9fb2"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/case_insensitive_map.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/common/string.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/common/helper.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/common/enum_util.hpp

@@ -200,6 +200,8 @@ enum class PhysicalType : uint8_t;
 
 enum class PragmaType : uint8_t;
 
+enum class PreparedParamType : uint8_t;
+
 enum class ProfilerPrintFormat : uint8_t;
 
 enum class QueryNodeType : uint8_t;
@@ -533,6 +535,9 @@ const char* EnumUtil::ToChars<PhysicalType>(PhysicalType value);
 template<>
 const char* EnumUtil::ToChars<PragmaType>(PragmaType value);
 
+template<>
+const char* EnumUtil::ToChars<PreparedParamType>(PreparedParamType value);
+
 template<>
 const char* EnumUtil::ToChars<ProfilerPrintFormat>(ProfilerPrintFormat value);
 
@@ -906,6 +911,9 @@ PhysicalType EnumUtil::FromString<PhysicalType>(const char *value);
 template<>
 PragmaType EnumUtil::FromString<PragmaType>(const char *value);
 
+template<>
+PreparedParamType EnumUtil::FromString<PreparedParamType>(const char *value);
+
 template<>
 ProfilerPrintFormat EnumUtil::FromString<ProfilerPrintFormat>(const char *value);