duckdb 0.8.2-dev3204.0 → 0.8.2-dev3250.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev3204.0",
+  "version": "0.8.2-dev3250.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/common/types/value.cpp

@@ -663,6 +663,20 @@ Value Value::MAP(const LogicalType &child_type, vector<Value> values) {
 
 	result.type_ = LogicalType::MAP(child_type);
 	result.is_null = false;
+	for (auto &val : values) {
+		D_ASSERT(val.type().InternalType() == PhysicalType::STRUCT);
+		auto &children = StructValue::GetChildren(val);
+
+		// Ensure that the field containing the keys is called 'key'
+		// and that the field containing the values is called 'value'
+		// this is required to make equality checks work
+		D_ASSERT(children.size() == 2);
+		child_list_t<Value> new_children;
+		new_children.reserve(2);
+		new_children.push_back(std::make_pair("key", children[0]));
+		new_children.push_back(std::make_pair("value", children[1]));
+		val = Value::STRUCT(std::move(new_children));
+	}
 	result.value_info_ = make_shared<NestedValueInfo>(std::move(values));
 	return result;
 }

package/src/duckdb/src/common/types/vector.cpp

@@ -1286,6 +1286,11 @@ void Vector::UTFVerify(idx_t count) {
 void Vector::VerifyMap(Vector &vector_p, const SelectionVector &sel_p, idx_t count) {
 #ifdef DEBUG
 	D_ASSERT(vector_p.GetType().id() == LogicalTypeId::MAP);
+	auto &child = ListType::GetChildType(vector_p.GetType());
+	D_ASSERT(StructType::GetChildCount(child) == 2);
+	D_ASSERT(StructType::GetChildName(child, 0) == "key");
+	D_ASSERT(StructType::GetChildName(child, 1) == "value");
+
 	auto valid_check = MapVector::CheckMapValidity(vector_p, count, sel_p);
 	D_ASSERT(valid_check == MapInvalidReason::VALID);
 #endif // DEBUG
@@ -1410,9 +1415,6 @@ void Vector::Verify(Vector &vector_p, const SelectionVector &sel_p, idx_t count)
 				}
 			}
 		}
-		if (vector->GetType().id() == LogicalTypeId::MAP) {
-			VerifyMap(*vector, *sel, count);
-		}
 
 		if (vector->GetType().id() == LogicalTypeId::UNION) {
 			VerifyUnion(*vector, *sel, count);
@@ -1460,6 +1462,10 @@ void Vector::Verify(Vector &vector_p, const SelectionVector &sel_p, idx_t count)
 			}
 			Vector::Verify(child, child_sel, child_count);
 		}
+
+		if (vector->GetType().id() == LogicalTypeId::MAP) {
+			VerifyMap(*vector, *sel, count);
+		}
 	}
 #endif
 }

package/src/duckdb/src/common/types.cpp

@@ -927,7 +927,24 @@ LogicalType LogicalType::AGGREGATE_STATE(aggregate_state_t state_type) { // NOLI
 //===--------------------------------------------------------------------===//
 // Map Type
 //===--------------------------------------------------------------------===//
-LogicalType LogicalType::MAP(const LogicalType &child) {
+LogicalType LogicalType::MAP(const LogicalType &child_p) {
+	D_ASSERT(child_p.id() == LogicalTypeId::STRUCT);
+	auto &children = StructType::GetChildTypes(child_p);
+	D_ASSERT(children.size() == 2);
+
+	// We do this to enforce that for every MAP created, the keys are called "key"
+	// and the values are called "value"
+
+	// This is done because for Vector the keys of the STRUCT are used in equality checks.
+	// Vector::Reference will throw if the types don't match
+	child_list_t<LogicalType> new_children(2);
+	new_children[0] = children[0];
+	new_children[0].first = "key";
+
+	new_children[1] = children[1];
+	new_children[1].first = "value";
+
+	auto child = LogicalType::STRUCT(std::move(new_children));
 	auto info = make_shared<ListTypeInfo>(child);
 	return LogicalType(LogicalTypeId::MAP, std::move(info));
 }

package/src/duckdb/src/core_functions/aggregate/nested/histogram.cpp

@@ -142,6 +142,7 @@ static void HistogramFinalizeFunction(Vector &state_vector, AggregateInputData &
 		list_struct_data[rid].offset = old_len;
 		old_len += list_struct_data[rid].length;
 	}
+	result.Verify(count);
 }
 
 unique_ptr<FunctionData> HistogramBindFunction(ClientContext &context, AggregateFunction &function,

package/src/duckdb/src/core_functions/scalar/map/map_entries.cpp

@@ -42,8 +42,8 @@ static unique_ptr<FunctionData> MapEntriesBind(ClientContext &context, ScalarFun
 	auto &key_type = MapType::KeyType(map);
 	auto &value_type = MapType::ValueType(map);
 
-	child_types.push_back(make_pair("k", key_type));
-	child_types.push_back(make_pair("v", value_type));
+	child_types.push_back(make_pair("key", key_type));
+	child_types.push_back(make_pair("value", value_type));
 
 	auto row_type = LogicalType::STRUCT(child_types);
 

package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp

@@ -308,40 +308,37 @@ OperatorResultType PhysicalUnnest::ExecuteInternal(ExecutionContext &context, Da
 				// UNNEST(NULL)
 				chunk.SetCardinality(0);
 				break;
+			}
 
-			} else {
-
-				auto &vector_data = state.list_vector_data[col_idx];
-				auto current_idx = vector_data.sel->get_index(state.current_row);
-
-				if (!vector_data.validity.RowIsValid(current_idx)) {
-					UnnestNull(0, this_chunk_len, result_vector);
+			auto &vector_data = state.list_vector_data[col_idx];
+			auto current_idx = vector_data.sel->get_index(state.current_row);
 
-				} else {
+			if (!vector_data.validity.RowIsValid(current_idx)) {
+				UnnestNull(0, this_chunk_len, result_vector);
+				continue;
+			}
 
-					auto list_data = UnifiedVectorFormat::GetData<list_entry_t>(vector_data);
-					auto list_entry = list_data[current_idx];
+			auto list_data = UnifiedVectorFormat::GetData<list_entry_t>(vector_data);
+			auto list_entry = list_data[current_idx];
 
-					idx_t list_count = 0;
-					if (state.list_position < list_entry.length) {
-						// there are still list_count elements to unnest
-						list_count = MinValue<idx_t>(this_chunk_len, list_entry.length - state.list_position);
+			idx_t list_count = 0;
+			if (state.list_position < list_entry.length) {
+				// there are still list_count elements to unnest
+				list_count = MinValue<idx_t>(this_chunk_len, list_entry.length - state.list_position);
 
-						auto &list_vector = state.list_data.data[col_idx];
-						auto &child_vector = ListVector::GetEntry(list_vector);
-						auto list_size = ListVector::GetListSize(list_vector);
-						auto &child_vector_data = state.list_child_data[col_idx];
+				auto &list_vector = state.list_data.data[col_idx];
+				auto &child_vector = ListVector::GetEntry(list_vector);
+				auto list_size = ListVector::GetListSize(list_vector);
+				auto &child_vector_data = state.list_child_data[col_idx];
 
-						auto base_offset = list_entry.offset + state.list_position;
-						UnnestVector(child_vector_data, child_vector, list_size, base_offset, base_offset + list_count,
-						             result_vector);
-					}
+				auto base_offset = list_entry.offset + state.list_position;
+				UnnestVector(child_vector_data, child_vector, list_size, base_offset, base_offset + list_count,
+				             result_vector);
+			}
 
-					// fill the rest with NULLs
-					if (list_count != this_chunk_len) {
-						UnnestNull(list_count, this_chunk_len, result_vector);
-					}
-				}
+			// fill the rest with NULLs
+			if (list_count != this_chunk_len) {
+				UnnestNull(list_count, this_chunk_len, result_vector);
 			}
 		}
 

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev3204"
+#define DUCKDB_VERSION "0.8.2-dev3250"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "ade3443f4a"
+#define DUCKDB_SOURCE_ID "a7f85b3b2c"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -68,7 +68,7 @@ struct DebugForceNoCrossProduct {
 struct OrderedAggregateThreshold {
 	static constexpr const char *Name = "ordered_aggregate_threshold"; // NOLINT
 	static constexpr const char *Description =                         // NOLINT
-	    "the number of rows to accumulate before sorting, used for tuning";
+	    "The number of rows to accumulate before sorting, used for tuning";
 	static constexpr const LogicalTypeId InputType = LogicalTypeId::UBIGINT; // NOLINT
 	static void SetLocal(ClientContext &context, const Value &parameter);
 	static void ResetLocal(ClientContext &context);
@@ -225,6 +225,7 @@ struct EnableProgressBarSetting {
 	static void ResetLocal(ClientContext &context);
 	static Value GetSetting(ClientContext &context);
 };
+
 struct EnableProgressBarPrintSetting {
 	static constexpr const char *Name = "enable_progress_bar_print";
 	static constexpr const char *Description =
@@ -235,15 +236,6 @@ struct EnableProgressBarPrintSetting {
 	static Value GetSetting(ClientContext &context);
 };
 
-struct ExperimentalParallelCSVSetting {
-	static constexpr const char *Name = "experimental_parallel_csv";
-	static constexpr const char *Description = "Whether or not to use the experimental parallel CSV reader";
-	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
-	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
-	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
-	static Value GetSetting(ClientContext &context);
-};
-
 struct ExplainOutputSetting {
 	static constexpr const char *Name = "explain_output";
 	static constexpr const char *Description = "Output of EXPLAIN statements (ALL, OPTIMIZED_ONLY, PHYSICAL_ONLY)";

package/src/duckdb/src/main/capi/arrow-c.cpp

@@ -166,9 +166,8 @@ void EmptyStreamRelease(ArrowArrayStream *) {
 }
 
 void FactoryGetSchema(uintptr_t stream_factory_ptr, duckdb::ArrowSchemaWrapper &schema) {
-	auto private_data =
-	    reinterpret_cast<PrivateData *>(reinterpret_cast<ArrowArrayStream *>(stream_factory_ptr)->private_data);
-	schema.arrow_schema = *private_data->schema;
+	auto stream = reinterpret_cast<ArrowArrayStream *>(stream_factory_ptr);
+	stream->get_schema(stream, &schema.arrow_schema);
 
 	// Need to nullify the root schema's release function here, because streams don't allow us to set the release
 	// function. For the schema's children, we nullify the release functions in `duckdb_arrow_scan`, so we don't need to

package/src/duckdb/src/main/config.cpp

@@ -75,7 +75,6 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(EnableProfilingSetting),
                                                  DUCKDB_LOCAL(EnableProgressBarSetting),
                                                  DUCKDB_LOCAL(EnableProgressBarPrintSetting),
-                                                 DUCKDB_GLOBAL(ExperimentalParallelCSVSetting),
                                                  DUCKDB_LOCAL(ExplainOutputSetting),
                                                  DUCKDB_GLOBAL(ExtensionDirectorySetting),
                                                  DUCKDB_GLOBAL(ExternalThreadsSetting),

package/src/duckdb/src/main/settings/settings.cpp

@@ -564,21 +564,6 @@ Value EnableProgressBarPrintSetting::GetSetting(ClientContext &context) {
 	return Value::BOOLEAN(ClientConfig::GetConfig(context).print_progress_bar);
 }
 
-//===--------------------------------------------------------------------===//
-// Experimental Parallel CSV
-//===--------------------------------------------------------------------===//
-void ExperimentalParallelCSVSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
-	Printer::Print("experimental_parallel_csv is deprecated and will be removed with the next release - the parallel "
-	               "CSV reader is now standard and does not need to be manually enabled anymore 1");
-}
-
-void ExperimentalParallelCSVSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
-}
-
-Value ExperimentalParallelCSVSetting::GetSetting(ClientContext &context) {
-	return Value();
-}
-
 //===--------------------------------------------------------------------===//
 // Explain Output
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/storage/compression/fsst.cpp

@@ -461,7 +461,7 @@ void FSSTStorage::Compress(CompressionState &state_p, Vector &scan_vector, idx_t
 	    &sizes_in[0],         /* IN: byte-lengths of the inputs */
 	    &strings_in[0],       /* IN: input string start pointers. */
 	    compress_buffer_size, /* IN: byte-length of output buffer. */
-	    &compress_buffer[0],  /* OUT: memorxy buffer to put the compressed strings in (one after the other). */
+	    &compress_buffer[0],  /* OUT: memory buffer to put the compressed strings in (one after the other). */
 	    &sizes_out[0],        /* OUT: byte-lengths of the compressed strings. */
 	    &strings_out[0]       /* OUT: output string start pointers. Will all point into [output,output+size). */
 	);

package/src/duckdb/src/storage/data_table.cpp

@@ -1073,20 +1073,19 @@ void DataTable::VerifyUpdateConstraints(ClientContext &context, TableCatalogEntr
                                         const vector<PhysicalIndex> &column_ids) {
 	auto &constraints = table.GetConstraints();
 	auto &bound_constraints = table.GetBoundConstraints();
-	for (idx_t i = 0; i < bound_constraints.size(); i++) {
-		auto &base_constraint = constraints[i];
-		auto &constraint = bound_constraints[i];
+	for (idx_t constr_idx = 0; constr_idx < bound_constraints.size(); constr_idx++) {
+		auto &base_constraint = constraints[constr_idx];
+		auto &constraint = bound_constraints[constr_idx];
 		switch (constraint->type) {
 		case ConstraintType::NOT_NULL: {
 			auto &bound_not_null = *reinterpret_cast<BoundNotNullConstraint *>(constraint.get());
 			auto &not_null = *reinterpret_cast<NotNullConstraint *>(base_constraint.get());
 			// check if the constraint is in the list of column_ids
-			// FIXME: double usage of 'i'?
-			for (idx_t i = 0; i < column_ids.size(); i++) {
-				if (column_ids[i] == bound_not_null.index) {
+			for (idx_t col_idx = 0; col_idx < column_ids.size(); col_idx++) {
+				if (column_ids[col_idx] == bound_not_null.index) {
 					// found the column id: check the data in
 					auto &col = table.GetColumn(LogicalIndex(not_null.index));
-					VerifyNotNullConstraint(table, chunk.data[i], chunk.size(), col.Name());
+					VerifyNotNullConstraint(table, chunk.data[col_idx], chunk.size(), col.Name());
 					break;
 				}
 			}
@@ -1122,10 +1121,10 @@ void DataTable::VerifyUpdateConstraints(ClientContext &context, TableCatalogEntr
 void DataTable::Update(TableCatalogEntry &table, ClientContext &context, Vector &row_ids,
                        const vector<PhysicalIndex> &column_ids, DataChunk &updates) {
 	D_ASSERT(row_ids.GetType().InternalType() == ROW_TYPE);
-
 	D_ASSERT(column_ids.size() == updates.ColumnCount());
-	auto count = updates.size();
 	updates.Verify();
+
+	auto count = updates.size();
 	if (count == 0) {
 		return;
 	}
@@ -1138,24 +1137,36 @@ void DataTable::Update(TableCatalogEntry &table, ClientContext &context, Vector
 	VerifyUpdateConstraints(context, table, updates, column_ids);
 
 	// now perform the actual update
-	auto &transaction = DuckTransaction::Get(context, db);
+	Vector max_row_id_vec(Value::BIGINT(MAX_ROW_ID));
+	Vector row_ids_slice(LogicalType::BIGINT);
+	DataChunk updates_slice;
+	updates_slice.InitializeEmpty(updates.GetTypes());
 
-	updates.Flatten();
-	row_ids.Flatten(count);
-	auto ids = FlatVector::GetData<row_t>(row_ids);
-	auto first_id = FlatVector::GetValue<row_t>(row_ids, 0);
-	if (first_id >= MAX_ROW_ID) {
-		// update is in transaction-local storage: push update into local storage
-		auto &local_storage = LocalStorage::Get(context, db);
-		local_storage.Update(*this, row_ids, column_ids, updates);
-		return;
+	SelectionVector sel_local_update(count), sel_global_update(count);
+	auto n_local_update = VectorOperations::GreaterThanEquals(row_ids, max_row_id_vec, nullptr, count,
+	                                                          &sel_local_update, &sel_global_update);
+	auto n_global_update = count - n_local_update;
+
+	// row id > MAX_ROW_ID? transaction-local storage
+	if (n_local_update > 0) {
+		updates_slice.Slice(updates, sel_local_update, n_local_update);
+		updates_slice.Flatten();
+		row_ids_slice.Slice(row_ids, sel_local_update, n_local_update);
+		row_ids_slice.Flatten(n_local_update);
+
+		LocalStorage::Get(context, db).Update(*this, row_ids_slice, column_ids, updates_slice);
 	}
 
-	// update is in the row groups
-	// we need to figure out for each id to which row group it belongs
-	// usually all (or many) ids belong to the same row group
-	// we iterate over the ids and check for every id if it belongs to the same row group as their predecessor
-	row_groups->Update(transaction, ids, column_ids, updates);
+	// otherwise global storage
+	if (n_global_update > 0) {
+		updates_slice.Slice(updates, sel_global_update, n_global_update);
+		updates_slice.Flatten();
+		row_ids_slice.Slice(row_ids, sel_global_update, n_global_update);
+		row_ids_slice.Flatten(n_global_update);
+
+		row_groups->Update(DuckTransaction::Get(context, db), FlatVector::GetData<row_t>(row_ids_slice), column_ids,
+		                   updates_slice);
+	}
 }
 
 void DataTable::UpdateColumn(TableCatalogEntry &table, ClientContext &context, Vector &row_ids,