duckdb 0.8.2-dev5002.0 → 0.8.2-dev5080.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev5002.0",
+  "version": "0.8.2-dev5080.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/icu/icu-strptime.cpp

@@ -75,6 +75,7 @@ struct ICUStrptime : public ICUDateFunc {
 		calendar->set(UCAL_MINUTE, parsed.data[4]);
 		calendar->set(UCAL_SECOND, parsed.data[5]);
 		calendar->set(UCAL_MILLISECOND, parsed.data[6] / Interval::MICROS_PER_MSEC);
+		micros = parsed.data[6] % Interval::MICROS_PER_MSEC;
 
 		// This overrides the TZ setting, so only use it if an offset was parsed.
 		// Note that we don't bother/worry about the DST setting because the two just combine.

package/src/duckdb/extension/json/json_functions/copy_json.cpp

@@ -101,7 +101,7 @@ static BoundStatement CopyToJSONPlan(Binder &binder, CopyStatement &stmt) {
 	info.options["quote"] = {""};
 	info.options["escape"] = {""};
 	info.options["delimiter"] = {"\n"};
-	info.options["header"] = {0};
+	info.options["header"] = {{0}};
 
 	return binder.Bind(*stmt_copy);
 }

package/src/duckdb/src/common/enum_util.cpp

@@ -5503,6 +5503,8 @@ const char* EnumUtil::ToChars<StrTimeSpecifier>(StrTimeSpecifier value) {
 		return "LOCALE_APPROPRIATE_DATE";
 	case StrTimeSpecifier::LOCALE_APPROPRIATE_TIME:
 		return "LOCALE_APPROPRIATE_TIME";
+	case StrTimeSpecifier::NANOSECOND_PADDED:
+		return "NANOSECOND_PADDED";
 	default:
 		throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented", value));
 	}
@@ -5606,6 +5608,9 @@ StrTimeSpecifier EnumUtil::FromString<StrTimeSpecifier>(const char *value) {
 	if (StringUtil::Equals(value, "LOCALE_APPROPRIATE_TIME")) {
 		return StrTimeSpecifier::LOCALE_APPROPRIATE_TIME;
 	}
+	if (StringUtil::Equals(value, "NANOSECOND_PADDED")) {
+		return StrTimeSpecifier::NANOSECOND_PADDED;
+	}
 	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
 }
 

package/src/duckdb/src/common/types/data_chunk.cpp

@@ -263,7 +263,6 @@ void DataChunk::Deserialize(Deserializer &deserializer) {
 
 	// read and set the row count
 	auto row_count = deserializer.ReadProperty<sel_t>(100, "rows");
-	SetCardinality(row_count);
 
 	// read the types
 	vector<LogicalType> types;
@@ -275,6 +274,7 @@ void DataChunk::Deserialize(Deserializer &deserializer) {
 	// initialize the data chunk
 	D_ASSERT(!types.empty());
 	Initialize(Allocator::DefaultAllocator(), types);
+	SetCardinality(row_count);
 
 	// read the data
 	deserializer.ReadList(102, "columns", [&](Deserializer::List &list, idx_t i) {

package/src/duckdb/src/common/types.cpp

@@ -659,6 +659,10 @@ LogicalType LogicalType::MaxLogicalType(const LogicalType &left, const LogicalTy
 		return right;
 	} else if (right.id() == LogicalTypeId::UNKNOWN) {
 		return left;
+	} else if ((right.id() == LogicalTypeId::ENUM || left.id() == LogicalTypeId::ENUM) && right.id() != left.id()) {
+		// if one is an enum and the other is not, compare strings, not enums
+		// see https://github.com/duckdb/duckdb/issues/8561
+		return LogicalTypeId::VARCHAR;
 	} else if (left.id() < right.id()) {
 		return right;
 	}
@@ -911,6 +915,11 @@ const string &StructType::GetChildName(const LogicalType &type, idx_t index) {
 idx_t StructType::GetChildCount(const LogicalType &type) {
 	return StructType::GetChildTypes(type).size();
 }
+bool StructType::IsUnnamed(const LogicalType &type) {
+	auto &child_types = StructType::GetChildTypes(type);
+	D_ASSERT(child_types.size() > 0);
+	return child_types[0].first.empty();
+}
 
 LogicalType LogicalType::STRUCT(child_list_t<LogicalType> children) {
 	auto info = make_shared<StructTypeInfo>(std::move(children));

package/src/duckdb/src/core_functions/scalar/date/strftime.cpp

@@ -183,7 +183,7 @@ struct StrpTimeFunction {
 		auto &func_expr = state.expr.Cast<BoundFunctionExpression>();
 		auto &info = func_expr.bind_info->Cast<StrpTimeBindData>();
 
-		if (ConstantVector::IsNull(args.data[1])) {
+		if (args.data[1].GetVectorType() == VectorType::CONSTANT_VECTOR && ConstantVector::IsNull(args.data[1])) {
 			result.SetVectorType(VectorType::CONSTANT_VECTOR);
 			ConstantVector::SetNull(result, true);
 			return;
@@ -203,7 +203,7 @@ struct StrpTimeFunction {
 		auto &func_expr = state.expr.Cast<BoundFunctionExpression>();
 		auto &info = func_expr.bind_info->Cast<StrpTimeBindData>();
 
-		if (ConstantVector::IsNull(args.data[1])) {
+		if (args.data[1].GetVectorType() == VectorType::CONSTANT_VECTOR && ConstantVector::IsNull(args.data[1])) {
 			result.SetVectorType(VectorType::CONSTANT_VECTOR);
 			ConstantVector::SetNull(result, true);
 			return;

package/src/duckdb/src/core_functions/scalar/list/array_slice.cpp

@@ -237,6 +237,16 @@ static void ExecuteFlatSlice(Vector &result, Vector &list_vector, Vector &begin_
 		auto end_idx = end_data.sel->get_index(i);
 		auto step_idx = step_vector ? step_data.sel->get_index(i) : 0;
 
+		auto list_valid = list_data.validity.RowIsValid(list_idx);
+		auto begin_valid = begin_data.validity.RowIsValid(begin_idx);
+		auto end_valid = end_data.validity.RowIsValid(end_idx);
+		auto step_valid = step_vector && step_data.validity.RowIsValid(step_idx);
+
+		if (!list_valid || !begin_valid || !end_valid || (step_vector && !step_valid)) {
+			result_mask.SetInvalid(i);
+			continue;
+		}
+
 		auto sliced = reinterpret_cast<INPUT_TYPE *>(list_data.data)[list_idx];
 		auto begin = begin_is_empty ? 0 : reinterpret_cast<INDEX_TYPE *>(begin_data.data)[begin_idx];
 		auto end = end_is_empty ? ValueLength<INPUT_TYPE, INDEX_TYPE>(sliced)
@@ -248,23 +258,19 @@ static void ExecuteFlatSlice(Vector &result, Vector &list_vector, Vector &begin_
 			begin = end_is_empty ? 0 : begin;
 			end = begin_is_empty ? ValueLength<INPUT_TYPE, INDEX_TYPE>(sliced) : end;
 		}
-		auto list_valid = list_data.validity.RowIsValid(list_idx);
-		auto begin_valid = begin_data.validity.RowIsValid(begin_idx);
-		auto end_valid = end_data.validity.RowIsValid(end_idx);
-		auto step_valid = step_vector && step_data.validity.RowIsValid(step_idx);
 
 		bool clamp_result = false;
-		if (list_valid && begin_valid && end_valid && (step_valid || step == 1)) {
+		if (step_valid || step == 1) {
 			clamp_result = ClampSlice(sliced, begin, end);
 		}
 
 		auto length = 0;
-		if (step_vector && step_valid && list_valid && begin_valid && end_valid && end - begin > 0) {
+		if (end - begin > 0) {
 			length = CalculateSliceLength(begin, end, step, step_valid);
 		}
 		sel_length += length;
 
-		if (!list_valid || !begin_valid || !end_valid || (step_vector && !step_valid) || !clamp_result) {
+		if (!clamp_result) {
 			result_mask.SetInvalid(i);
 		} else if (!step_vector) {
 			result_data[i] = SliceValue<INPUT_TYPE, INDEX_TYPE>(result, sliced, begin, end);

package/src/duckdb/src/core_functions/scalar/struct/struct_pack.cpp

@@ -39,15 +39,21 @@ static unique_ptr<FunctionData> StructPackBind(ClientContext &context, ScalarFun
 		throw Exception("Can't pack nothing into a struct");
 	}
 	child_list_t<LogicalType> struct_children;
+	bool unnamed = false;
 	for (idx_t i = 0; i < arguments.size(); i++) {
 		auto &child = arguments[i];
-		if (child->alias.empty() && bound_function.name == "struct_pack") {
-			throw BinderException("Need named argument for struct pack, e.g. STRUCT_PACK(a := b)");
+		if (child->alias.empty()) {
+			if (bound_function.name == "struct_pack") {
+				throw BinderException("Need named argument for struct pack, e.g. STRUCT_PACK(a := b)");
+			} else {
+				D_ASSERT(bound_function.name == "row");
+				if (i > 1) {
+					D_ASSERT(unnamed);
+				}
+				unnamed = true;
+			}
 		}
-		if (child->alias.empty() && bound_function.name == "row") {
-			child->alias = "v" + std::to_string(i + 1);
-		}
-		if (name_collision_set.find(child->alias) != name_collision_set.end()) {
+		if (!child->alias.empty() && name_collision_set.find(child->alias) != name_collision_set.end()) {
 			throw BinderException("Duplicate struct entry name \"%s\"", child->alias);
 		}
 		name_collision_set.insert(child->alias);

package/src/duckdb/src/execution/operator/csv_scanner/parallel_csv_reader.cpp

@@ -335,8 +335,6 @@ normal : {
 		if (c == options.dialect_options.state_machine_options.delimiter) {
 			// delimiter: end the value and add it to the chunk
 			goto add_value;
-		} else if (c == options.dialect_options.state_machine_options.quote && try_add_line) {
-			return false;
 		} else if (StringUtil::CharacterIsNewline(c)) {
 			// newline: add row
 			if (column > 0 || try_add_line || parse_chunk.data.size() == 1) {

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp

@@ -141,6 +141,10 @@ struct SniffValue {
 		     (current_char == '\r' || current_char == '\n')) ||
 		    (machine.dialect_options.new_line == NewLineIdentifier::CARRY_ON && current_char == '\n')) {
 			machine.rows_read++;
+		}
+
+		if ((machine.previous_state == CSVState::RECORD_SEPARATOR && machine.state != CSVState::EMPTY_LINE) ||
+		    (machine.state != CSVState::RECORD_SEPARATOR && machine.previous_state == CSVState::CARRIAGE_RETURN)) {
 			sniffed_values[machine.cur_rows].position = machine.line_start_pos;
 			sniffed_values[machine.cur_rows].set = true;
 			machine.line_start_pos = current_pos;
@@ -287,11 +291,15 @@ void CSVSniffer::DetectTypes() {
 		candidate->csv_buffer_iterator.Process<SniffValue>(*candidate, tuples);
 		// Potentially Skip empty rows (I find this dirty, but it is what the original code does)
 		idx_t true_start = 0;
+		idx_t true_pos = 0;
 		idx_t values_start = 0;
 		while (true_start < tuples.size()) {
 			if (tuples[true_start].values.empty() ||
 			    (tuples[true_start].values.size() == 1 && tuples[true_start].values[0].IsNull())) {
 				true_start = tuples[true_start].line_number;
+				if (true_start < tuples.size()) {
+					true_pos = tuples[true_start].position;
+				}
 				values_start++;
 			} else {
 				break;
@@ -301,7 +309,11 @@ void CSVSniffer::DetectTypes() {
 		// Potentially Skip Notes (I also find this dirty, but it is what the original code does)
 		while (true_start < tuples.size()) {
 			if (tuples[true_start].values.size() < max_columns_found && !options.null_padding) {
+
 				true_start = tuples[true_start].line_number;
+				if (true_start < tuples.size()) {
+					true_pos = tuples[true_start].position;
+				}
 				values_start++;
 			} else {
 				break;
@@ -317,7 +329,7 @@ void CSVSniffer::DetectTypes() {
 			row_idx = 1;
 		}
 		if (!tuples.empty()) {
-			best_start_without_header = tuples[0].position;
+			best_start_without_header = tuples[0].position - true_pos;
 		}
 
 		// First line where we start our type detection
@@ -387,7 +399,7 @@ void CSVSniffer::DetectTypes() {
 			best_sql_types_candidates_per_column_idx = info_sql_types_candidates;
 			best_format_candidates = format_candidates;
 			best_header_row = tuples[0].values;
-			best_start_with_header = tuples[0].position;
+			best_start_with_header = tuples[0].position - true_pos;
 		}
 	}
 	// Assert that it's all good at this point.

package/src/duckdb/src/execution/operator/persistent/physical_export.cpp

@@ -59,7 +59,7 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info,
 	if (info.format == "csv") {
 		// insert default csv options, if not specified
 		if (info.options.find("header") == info.options.end()) {
-			info.options["header"].push_back(Value::INTEGER(0));
+			info.options["header"].push_back(Value::INTEGER(1));
 		}
 		if (info.options.find("delimiter") == info.options.end() && info.options.find("sep") == info.options.end() &&
 		    info.options.find("delim") == info.options.end()) {

package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp

@@ -184,19 +184,21 @@ static bool PlanIndexJoin(ClientContext &context, LogicalComparisonJoin &op, uni
 	if (!index) {
 		return false;
 	}
-	// index joins are not supported if there are pushed down table filters
-	D_ASSERT(right->type == PhysicalOperatorType::TABLE_SCAN);
-	auto &tbl_scan = right->Cast<PhysicalTableScan>();
-	//	if (tbl_scan.table_filters && !tbl_scan.table_filters->filters.empty()) {
-	//		return false;
-	//	}
+
 	// index joins are disabled if enable_optimizer is false
 	if (!ClientConfig::GetConfig(context).enable_optimizer) {
 		return false;
 	}
+
+	// index joins are disabled on default
+	auto force_index_join = ClientConfig::GetConfig(context).force_index_join;
+	if (!ClientConfig::GetConfig(context).enable_index_join && !force_index_join) {
+		return false;
+	}
+
 	// check if the cardinality difference justifies an index join
-	if (!((ClientConfig::GetConfig(context).force_index_join ||
-	       left->estimated_cardinality < 0.01 * right->estimated_cardinality))) {
+	auto index_join_is_applicable = left->estimated_cardinality < 0.01 * right->estimated_cardinality;
+	if (!index_join_is_applicable && !force_index_join) {
 		return false;
 	}
 
@@ -205,6 +207,9 @@ static bool PlanIndexJoin(ClientContext &context, LogicalComparisonJoin &op, uni
 		swap(op.conditions[0].left, op.conditions[0].right);
 		swap(op.left_projection_map, op.right_projection_map);
 	}
+	D_ASSERT(right->type == PhysicalOperatorType::TABLE_SCAN);
+	auto &tbl_scan = right->Cast<PhysicalTableScan>();
+
 	plan = make_uniq<PhysicalIndexJoin>(op, std::move(left), std::move(right), std::move(op.conditions), op.join_type,
 	                                    op.left_projection_map, op.right_projection_map, tbl_scan.column_ids, *index,
 	                                    !swap_condition, op.estimated_cardinality);

package/src/duckdb/src/function/cast/struct_cast.cpp

@@ -9,10 +9,18 @@ unique_ptr<BoundCastData> StructBoundCastData::BindStructToStructCast(BindCastIn
 	vector<BoundCastInfo> child_cast_info;
 	auto &source_child_types = StructType::GetChildTypes(source);
 	auto &result_child_types = StructType::GetChildTypes(target);
+
+	auto target_is_unnamed = StructType::IsUnnamed(target);
+	auto source_is_unnamed = StructType::IsUnnamed(source);
+
 	if (source_child_types.size() != result_child_types.size()) {
 		throw TypeMismatchException(source, target, "Cannot cast STRUCTs of different size");
 	}
 	for (idx_t i = 0; i < source_child_types.size(); i++) {
+		if (!target_is_unnamed && !source_is_unnamed &&
+		    !StringUtil::CIEquals(source_child_types[i].first, result_child_types[i].first)) {
+			throw TypeMismatchException(source, target, "Cannot cast STRUCTs with different names");
+		}
 		auto child_cast = input.GetCastFunction(source_child_types[i].second, result_child_types[i].second);
 		child_cast_info.push_back(std::move(child_cast));
 	}

package/src/duckdb/src/function/cast/union_casts.cpp

@@ -304,6 +304,11 @@ static bool UnionToVarcharCast(Vector &source, Vector &result, idx_t count, Cast
 	// now construct the actual varchar vector
 	varchar_union.Flatten(count);
 	auto &tag_vector = UnionVector::GetTags(source);
+	auto tag_vector_type = tag_vector.GetVectorType();
+	if (tag_vector_type != VectorType::CONSTANT_VECTOR && tag_vector_type != VectorType::FLAT_VECTOR) {
+		tag_vector.Flatten(count);
+	}
+
 	auto tags = FlatVector::GetData<union_tag_t>(tag_vector);
 
 	auto &validity = FlatVector::Validity(varchar_union);

package/src/duckdb/src/function/function_binder.cpp

@@ -268,8 +268,17 @@ unique_ptr<Expression> FunctionBinder::BindScalarFunction(ScalarFunctionCatalogE
 
 	if (bound_function.null_handling == FunctionNullHandling::DEFAULT_NULL_HANDLING) {
 		for (auto &child : children) {
-			if (child->return_type == LogicalTypeId::SQLNULL ||
-			    (child->IsFoldable() && ExpressionExecutor::EvaluateScalar(context, *child).IsNull())) {
+			if (child->return_type == LogicalTypeId::SQLNULL) {
+				return make_uniq<BoundConstantExpression>(Value(LogicalType::SQLNULL));
+			}
+			if (!child->IsFoldable()) {
+				continue;
+			}
+			Value result;
+			if (!ExpressionExecutor::TryEvaluateScalar(context, *child, result)) {
+				continue;
+			}
+			if (result.IsNull()) {
 				return make_uniq<BoundConstantExpression>(Value(LogicalType::SQLNULL));
 			}
 		}

package/src/duckdb/src/function/pragma/pragma_functions.cpp

@@ -80,6 +80,10 @@ static void PragmaEnableForceParallelism(ClientContext &context, const FunctionP
 	ClientConfig::GetConfig(context).verify_parallelism = true;
 }
 
+static void PragmaEnableIndexJoin(ClientContext &context, const FunctionParameters &parameters) {
+	ClientConfig::GetConfig(context).enable_index_join = true;
+}
+
 static void PragmaEnableForceIndexJoin(ClientContext &context, const FunctionParameters &parameters) {
 	ClientConfig::GetConfig(context).force_index_join = true;
 }
@@ -140,6 +144,7 @@ void PragmaFunctions::RegisterFunction(BuiltinFunctions &set) {
 	set.AddFunction(PragmaFunction::PragmaStatement("enable_optimizer", PragmaEnableOptimizer));
 	set.AddFunction(PragmaFunction::PragmaStatement("disable_optimizer", PragmaDisableOptimizer));
 
+	set.AddFunction(PragmaFunction::PragmaStatement("enable_index_join", PragmaEnableIndexJoin));
 	set.AddFunction(PragmaFunction::PragmaStatement("force_index_join", PragmaEnableForceIndexJoin));
 	set.AddFunction(PragmaFunction::PragmaStatement("force_checkpoint", PragmaForceCheckpoint));
 

package/src/duckdb/src/function/scalar/strftime_format.cpp

@@ -26,6 +26,8 @@ idx_t StrfTimepecifierSize(StrTimeSpecifier specifier) {
 	case StrTimeSpecifier::WEEK_NUMBER_PADDED_SUN_FIRST:
 	case StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST:
 		return 2;
+	case StrTimeSpecifier::NANOSECOND_PADDED:
+		return 9;
 	case StrTimeSpecifier::MICROSECOND_PADDED:
 		return 6;
 	case StrTimeSpecifier::MILLISECOND_PADDED:
@@ -183,9 +185,15 @@ char *StrfTimeFormat::WritePadded3(char *target, uint32_t value) {
 	}
 }
 
-// write a value in the range of 0..999999 padded to 6 digits
+// write a value in the range of 0..999999... padded to the given number of digits
 char *StrfTimeFormat::WritePadded(char *target, uint32_t value, size_t padding) {
-	D_ASSERT(padding % 2 == 0);
+	D_ASSERT(padding > 1);
+	if (padding % 2) {
+		int decimals = value % 1000;
+		WritePadded3(target + padding - 3, decimals);
+		value /= 1000;
+		padding -= 3;
+	}
 	for (size_t i = 0; i < padding / 2; i++) {
 		int decimals = value % 100;
 		WritePadded2(target + padding - 2 * (i + 1), decimals);
@@ -309,11 +317,14 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t
 	case StrTimeSpecifier::SECOND_PADDED:
 		target = WritePadded2(target, data[5]);
 		break;
+	case StrTimeSpecifier::NANOSECOND_PADDED:
+		target = WritePadded(target, data[6] * Interval::NANOS_PER_MICRO, 9);
+		break;
 	case StrTimeSpecifier::MICROSECOND_PADDED:
 		target = WritePadded(target, data[6], 6);
 		break;
 	case StrTimeSpecifier::MILLISECOND_PADDED:
-		target = WritePadded3(target, data[6] / 1000);
+		target = WritePadded3(target, data[6] / Interval::MICROS_PER_MSEC);
 		break;
 	case StrTimeSpecifier::UTC_OFFSET: {
 		*target++ = (data[7] < 0) ? '-' : '+';
@@ -516,6 +527,9 @@ string StrTimeFormat::ParseFormatSpecifier(const string &format_string, StrTimeF
 				case 'S':
 					specifier = StrTimeSpecifier::SECOND_PADDED;
 					break;
+				case 'n':
+					specifier = StrTimeSpecifier::NANOSECOND_PADDED;
+					break;
 				case 'f':
 					specifier = StrTimeSpecifier::MICROSECOND_PADDED;
 					break;
@@ -660,6 +674,8 @@ int StrpTimeFormat::NumericSpecifierWidth(StrTimeSpecifier specifier) {
 		return 4;
 	case StrTimeSpecifier::MICROSECOND_PADDED:
 		return 6;
+	case StrTimeSpecifier::NANOSECOND_PADDED:
+		return 9;
 	default:
 		return -1;
 	}
@@ -855,15 +871,20 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const {
 				// seconds
 				result_data[5] = number;
 				break;
+			case StrTimeSpecifier::NANOSECOND_PADDED:
+				D_ASSERT(number < Interval::NANOS_PER_SEC); // enforced by the length of the number
+				// microseconds (rounded)
+				result_data[6] = (number + Interval::NANOS_PER_MICRO / 2) / Interval::NANOS_PER_MICRO;
+				break;
 			case StrTimeSpecifier::MICROSECOND_PADDED:
-				D_ASSERT(number < 1000000ULL); // enforced by the length of the number
-				// milliseconds
+				D_ASSERT(number < Interval::MICROS_PER_SEC); // enforced by the length of the number
+				// microseconds
 				result_data[6] = number;
 				break;
 			case StrTimeSpecifier::MILLISECOND_PADDED:
-				D_ASSERT(number < 1000ULL); // enforced by the length of the number
-				// milliseconds
-				result_data[6] = number * 1000;
+				D_ASSERT(number < Interval::MSECS_PER_SEC); // enforced by the length of the number
+				// microseconds
+				result_data[6] = number * Interval::MICROS_PER_MSEC;
 				break;
 			case StrTimeSpecifier::WEEK_NUMBER_PADDED_SUN_FIRST:
 			case StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST:

package/src/duckdb/src/function/table/arrow.cpp

@@ -237,6 +237,10 @@ void ArrowTableFunction::PopulateArrowTableType(ArrowTableType &arrow_table, Arr
 
 unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBind(ClientContext &context, TableFunctionBindInput &input,
                                                            vector<LogicalType> &return_types, vector<string> &names) {
+	if (input.inputs[0].IsNull() || input.inputs[1].IsNull() || input.inputs[2].IsNull()) {
+		throw BinderException("arrow_scan: pointers cannot be null");
+	}
+
 	auto stream_factory_ptr = input.inputs[0].GetPointer();
 	auto stream_factory_produce = (stream_factory_produce_t)input.inputs[1].GetPointer();       // NOLINT
 	auto stream_factory_get_schema = (stream_factory_get_schema_t)input.inputs[2].GetPointer(); // NOLINT

package/src/duckdb/src/function/table/copy_csv.cpp

@@ -157,6 +157,7 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, CopyInfo &in
 	}
 
 	bind_data->FinalizeRead(context);
+
 	if (options.auto_detect) {
 		// We must run the sniffer.
 		auto file_handle = BaseCSVReader::OpenCSV(context, options);
@@ -332,7 +333,7 @@ static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &co
 		global_data->WriteData(options.prefix.c_str(), options.prefix.size());
 	}
 
-	if (options.dialect_options.header) {
+	if (!(options.has_header && !options.dialect_options.header)) {
 		MemoryStream stream;
 		// write the header line to the file
 		for (idx_t i = 0; i < csv_data.options.name_list.size(); i++) {

package/src/duckdb/src/function/table/read_csv.cpp

@@ -178,7 +178,10 @@ public:
 		current_file_path = files_path_p[0];
 		CSVFileHandle *file_handle_ptr;
 
-		if (!buffer_manager) {
+		if (!buffer_manager || (options.skip_rows_set && options.dialect_options.skip_rows > 0)) {
+			// If our buffers are too small, and we skip too many rows there is a chance things will go over-buffer
+			// for now don't reuse the buffer manager
+			buffer_manager.reset();
 			file_handle = ReadCSV::OpenCSV(current_file_path, options.compression, context);
 			file_handle_ptr = file_handle.get();
 		} else {

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev5002"
+#define DUCKDB_VERSION "0.8.2-dev5080"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "239f51293c"
+#define DUCKDB_SOURCE_ID "e07b94a748"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/types.hpp

@@ -408,6 +408,7 @@ struct StructType {
 	DUCKDB_API static const LogicalType &GetChildType(const LogicalType &type, idx_t index);
 	DUCKDB_API static const string &GetChildName(const LogicalType &type, idx_t index);
 	DUCKDB_API static idx_t GetChildCount(const LogicalType &type);
+	DUCKDB_API static bool IsUnnamed(const LogicalType &type);
 };
 
 struct MapType {

package/src/duckdb/src/include/duckdb/function/scalar/strftime_format.hpp

@@ -52,7 +52,8 @@ enum class StrTimeSpecifier : uint8_t {
 	LOCALE_APPROPRIATE_DATE_AND_TIME =
 	    29, // %c - Locale’s appropriate date and time representation. (Mon Sep 30 07:06:05 2013)
 	LOCALE_APPROPRIATE_DATE = 30, // %x - Locale’s appropriate date representation. (09/30/13)
-	LOCALE_APPROPRIATE_TIME = 31  // %X - Locale’s appropriate time representation. (07:06:05)
+	LOCALE_APPROPRIATE_TIME = 31, // %X - Locale’s appropriate time representation. (07:06:05)
+	NANOSECOND_PADDED = 32 // %n - Nanosecond as a decimal number, zero-padded on the left. (000000000 - 999999999)
 };
 
 struct StrTimeFormat {

package/src/duckdb/src/include/duckdb/main/client_config.hpp

@@ -67,6 +67,8 @@ struct ClientConfig {
 	bool enable_caching_operators = true;
 	//! Force parallelism of small tables, used for testing
 	bool verify_parallelism = false;
+	//! Enable the optimizer to consider index joins, which are disabled on default
+	bool enable_index_join = false;
 	//! Force index join independent of table cardinality, used for testing
 	bool force_index_join = false;
 	//! Force out-of-core computation for operators that support it, used for testing

package/src/duckdb/src/include/duckdb/main/prepared_statement_data.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/common/winapi.hpp"
 #include "duckdb/planner/expression/bound_parameter_data.hpp"
+#include "duckdb/planner/bound_parameter_map.hpp"
 
 namespace duckdb {
 class CatalogEntry;

package/src/duckdb/src/include/duckdb/planner/bound_parameter_map.hpp

@@ -10,12 +10,39 @@
 
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/unordered_map.hpp"
+#include "duckdb/planner/expression/bound_parameter_data.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 
 namespace duckdb {
 
-struct BoundParameterData;
+class ParameterExpression;
+class BoundParameterExpression;
 
 using bound_parameter_map_t = case_insensitive_map_t<shared_ptr<BoundParameterData>>;
 
+struct BoundParameterMap {
+public:
+	explicit BoundParameterMap(case_insensitive_map_t<BoundParameterData> &parameter_data);
+
+public:
+	LogicalType GetReturnType(const string &identifier);
+
+	bound_parameter_map_t *GetParametersPtr();
+
+	const bound_parameter_map_t &GetParameters();
+
+	const case_insensitive_map_t<BoundParameterData> &GetParameterData();
+
+	unique_ptr<BoundParameterExpression> BindParameterExpression(ParameterExpression &expr);
+
+private:
+	shared_ptr<BoundParameterData> CreateOrGetData(const string &identifier);
+	void CreateNewParameter(const string &id, const shared_ptr<BoundParameterData> &param_data);
+
+private:
+	bound_parameter_map_t parameters;
+	// Pre-provided parameter data if populated
+	case_insensitive_map_t<BoundParameterData> &parameter_data;
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_data.hpp

@@ -9,7 +9,6 @@
 #pragma once
 
 #include "duckdb/common/types/value.hpp"
-#include "duckdb/planner/bound_parameter_map.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 
 namespace duckdb {
@@ -40,21 +39,4 @@ public:
 	static shared_ptr<BoundParameterData> Deserialize(Deserializer &deserializer);
 };
 
-struct BoundParameterMap {
-	explicit BoundParameterMap(case_insensitive_map_t<BoundParameterData> &parameter_data)
-	    : parameter_data(parameter_data) {
-	}
-
-	bound_parameter_map_t parameters;
-	case_insensitive_map_t<BoundParameterData> &parameter_data;
-
-	LogicalType GetReturnType(const string &identifier) {
-		auto it = parameter_data.find(identifier);
-		if (it == parameter_data.end()) {
-			return LogicalTypeId::UNKNOWN;
-		}
-		return it->second.return_type;
-	}
-};
-
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_expression.hpp

@@ -9,7 +9,7 @@
 #pragma once
 
 #include "duckdb/planner/expression.hpp"
-#include "duckdb/planner/expression/bound_parameter_data.hpp"
+#include "duckdb/planner/bound_parameter_map.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/planner/planner.hpp

@@ -11,7 +11,7 @@
 #include "duckdb/parser/sql_statement.hpp"
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/logical_operator.hpp"
-#include "duckdb/planner/expression/bound_parameter_data.hpp"
+#include "duckdb/planner/bound_parameter_map.hpp"
 
 namespace duckdb {
 class ClientContext;

package/src/duckdb/src/main/capi/prepared-c.cpp

@@ -124,10 +124,16 @@ duckdb_type duckdb_param_type(duckdb_prepared_statement prepared_statement, idx_
 	}
 	LogicalType param_type;
 	auto identifier = std::to_string(param_idx);
-	if (!wrapper->statement->data->TryGetType(identifier, param_type)) {
-		return DUCKDB_TYPE_INVALID;
+	if (wrapper->statement->data->TryGetType(identifier, param_type)) {
+		return ConvertCPPTypeToC(param_type);
+	}
+	// The value_map is gone after executing the prepared statement
+	// See if this is the case and we still have a value registered for it
+	auto it = wrapper->values.find(identifier);
+	if (it != wrapper->values.end()) {
+		return ConvertCPPTypeToC(it->second.type());
 	}
-	return ConvertCPPTypeToC(param_type);
+	return DUCKDB_TYPE_INVALID;
 }
 
 duckdb_state duckdb_clear_bindings(duckdb_prepared_statement prepared_statement) {

package/src/duckdb/src/planner/binder/expression/bind_comparison_expression.cpp

@@ -119,6 +119,7 @@ BindResult ExpressionBinder::BindExpression(ComparisonExpression &expr, idx_t de
 	if (!error.empty()) {
 		return BindResult(error);
 	}
+
 	// the children have been successfully resolved
 	auto &left = BoundExpression::GetExpression(*expr.left);
 	auto &right = BoundExpression::GetExpression(*expr.right);

package/src/duckdb/src/planner/binder/expression/bind_parameter_expression.cpp

@@ -7,35 +7,25 @@
 namespace duckdb {
 
 BindResult ExpressionBinder::BindExpression(ParameterExpression &expr, idx_t depth) {
-	auto bound_parameter = make_uniq<BoundParameterExpression>(expr.identifier);
-	bound_parameter->alias = expr.alias;
 	if (!binder.parameters) {
 		throw BinderException("Unexpected prepared parameter. This type of statement can't be prepared!");
 	}
 	auto parameter_id = expr.identifier;
-	// check if a parameter value has already been supplied
-	if (binder.parameters->parameter_data.count(parameter_id)) {
+
+	D_ASSERT(binder.parameters);
+	// Check if a parameter value has already been supplied
+	auto &parameter_data = binder.parameters->GetParameterData();
+	auto param_data_it = parameter_data.find(parameter_id);
+	if (param_data_it != parameter_data.end()) {
 		// it has! emit a constant directly
-		auto &data = binder.parameters->parameter_data[parameter_id];
+		auto &data = param_data_it->second;
 		auto constant = make_uniq<BoundConstantExpression>(data.GetValue());
 		constant->alias = expr.alias;
+		constant->return_type = binder.parameters->GetReturnType(parameter_id);
 		return BindResult(std::move(constant));
 	}
 
-	auto entry = binder.parameters->parameters.find(parameter_id);
-	if (entry == binder.parameters->parameters.end()) {
-		// no entry yet: create a new one
-		auto data = make_shared<BoundParameterData>();
-		data->return_type = binder.parameters->GetReturnType(parameter_id);
-		bound_parameter->return_type = data->return_type;
-		bound_parameter->parameter_data = data;
-		binder.parameters->parameters[parameter_id] = std::move(data);
-	} else {
-		// a prepared statement with this parameter index was already there: use it
-		auto &data = entry->second;
-		bound_parameter->parameter_data = data;
-		bound_parameter->return_type = binder.parameters->GetReturnType(parameter_id);
-	}
+	auto bound_parameter = binder.parameters->BindParameterExpression(expr);
 	return BindResult(std::move(bound_parameter));
 }
 

package/src/duckdb/src/planner/bound_parameter_map.cpp

@@ -0,0 +1,67 @@
+#include "duckdb/planner/bound_parameter_map.hpp"
+#include "duckdb/parser/expression/parameter_expression.hpp"
+#include "duckdb/planner/expression/bound_parameter_expression.hpp"
+
+namespace duckdb {
+
+BoundParameterMap::BoundParameterMap(case_insensitive_map_t<BoundParameterData> &parameter_data)
+    : parameter_data(parameter_data) {
+}
+
+LogicalType BoundParameterMap::GetReturnType(const string &identifier) {
+	D_ASSERT(!identifier.empty());
+	auto it = parameter_data.find(identifier);
+	if (it == parameter_data.end()) {
+		return LogicalTypeId::UNKNOWN;
+	}
+	return it->second.return_type;
+}
+
+bound_parameter_map_t *BoundParameterMap::GetParametersPtr() {
+	return &parameters;
+}
+
+const bound_parameter_map_t &BoundParameterMap::GetParameters() {
+	return parameters;
+}
+
+const case_insensitive_map_t<BoundParameterData> &BoundParameterMap::GetParameterData() {
+	return parameter_data;
+}
+
+shared_ptr<BoundParameterData> BoundParameterMap::CreateOrGetData(const string &identifier) {
+	auto entry = parameters.find(identifier);
+	if (entry == parameters.end()) {
+		// no entry yet: create a new one
+		auto data = make_shared<BoundParameterData>();
+		data->return_type = GetReturnType(identifier);
+
+		CreateNewParameter(identifier, data);
+		return data;
+	}
+	return entry->second;
+}
+
+unique_ptr<BoundParameterExpression> BoundParameterMap::BindParameterExpression(ParameterExpression &expr) {
+	auto &identifier = expr.identifier;
+	auto return_type = GetReturnType(identifier);
+
+	D_ASSERT(!parameter_data.count(identifier));
+
+	// No value has been supplied yet,
+	// We return a shared pointer to an object that will get populated wtih a Value later
+	// When the BoundParameterExpression get executed, this will be used to get the corresponding value
+	auto param_data = CreateOrGetData(identifier);
+	auto bound_expr = make_uniq<BoundParameterExpression>(identifier);
+	bound_expr->parameter_data = param_data;
+	bound_expr->return_type = return_type;
+	bound_expr->alias = expr.alias;
+	return bound_expr;
+}
+
+void BoundParameterMap::CreateNewParameter(const string &id, const shared_ptr<BoundParameterData> &param_data) {
+	D_ASSERT(!parameters.count(id));
+	parameters.emplace(std::make_pair(id, param_data));
+}
+
+} // namespace duckdb

package/src/duckdb/src/planner/planner.cpp

@@ -78,10 +78,10 @@ void Planner::CreatePlan(SQLStatement &statement) {
 	this->properties.parameter_count = parameter_count;
 	properties.bound_all_parameters = parameters_resolved;
 
-	Planner::VerifyPlan(context, plan, &bound_parameters.parameters);
+	Planner::VerifyPlan(context, plan, bound_parameters.GetParametersPtr());
 
 	// set up a map of parameter number -> value entries
-	for (auto &kv : bound_parameters.parameters) {
+	for (auto &kv : bound_parameters.GetParameters()) {
 		auto &identifier = kv.first;
 		auto &param = kv.second;
 		// check if the type of the parameter could be resolved

package/src/duckdb/src/storage/table/struct_column_data.cpp

@@ -15,6 +15,9 @@ StructColumnData::StructColumnData(BlockManager &block_manager, DataTableInfo &i
 	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
 	auto &child_types = StructType::GetChildTypes(type);
 	D_ASSERT(child_types.size() > 0);
+	if (type.id() != LogicalTypeId::UNION && StructType::IsUnnamed(type)) {
+		throw InvalidInputException("A table cannot be created from an unnamed struct");
+	}
 	// the sub column index, starting at 1 (0 is the validity mask)
 	idx_t sub_column_index = 1;
 	for (auto &child_type : child_types) {

package/src/duckdb/ub_src_planner.cpp

@@ -1,5 +1,7 @@
 #include "src/planner/bound_result_modifier.cpp"
 
+#include "src/planner/bound_parameter_map.cpp"
+
 #include "src/planner/expression_iterator.cpp"
 
 #include "src/planner/expression.cpp"