duckdb 0.8.2-dev145.0 → 0.8.2-dev1493.0

package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp

@@ -16,15 +16,6 @@ PartitionedTupleData::PartitionedTupleData(const PartitionedTupleData &other)
     : type(other.type), buffer_manager(other.buffer_manager), layout(other.layout.Copy()) {
 }
 
-unique_ptr<PartitionedTupleData> PartitionedTupleData::CreateShared() {
-	switch (type) {
-	case PartitionedTupleDataType::RADIX:
-		return make_uniq<RadixPartitionedTupleData>(Cast<RadixPartitionedTupleData>());
-	default:
-		throw NotImplementedException("CreateShared for this type of PartitionedTupleData");
-	}
-}
-
 PartitionedTupleData::~PartitionedTupleData() {
 }
 
@@ -233,6 +224,9 @@ void PartitionedTupleData::Combine(PartitionedTupleData &other) {
 }
 
 void PartitionedTupleData::Partition(TupleDataCollection &source, TupleDataPinProperties properties) {
+	if (source.Count() == 0) {
+		return;
+	}
 #ifdef DEBUG
 	const auto count_before = source.Count();
 #endif

package/src/duckdb/src/common/types/row/tuple_data_collection.cpp

@@ -220,6 +220,8 @@ void TupleDataCollection::AppendUnified(TupleDataPinState &pin_state, TupleDataC
 
 static inline void ToUnifiedFormatInternal(TupleDataVectorFormat &format, Vector &vector, const idx_t count) {
 	vector.ToUnifiedFormat(count, format.data);
+	format.original_sel = format.data.sel;
+	format.original_owned_sel.Initialize(format.data.owned_sel);
 	switch (vector.GetType().InternalType()) {
 	case PhysicalType::STRUCT: {
 		auto &entries = StructVector::GetEntries(vector);

package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp

@@ -296,7 +296,7 @@ static void ApplySliceRecursive(const Vector &source_v, TupleDataVectorFormat &s
 	D_ASSERT(source_format.combined_list_data);
 	auto &combined_list_data = *source_format.combined_list_data;
 
-	combined_list_data.selection_data = source_format.data.sel->Slice(combined_sel, count);
+	combined_list_data.selection_data = source_format.original_sel->Slice(combined_sel, count);
 	source_format.data.owned_sel.Initialize(combined_list_data.selection_data);
 	source_format.data.sel = &source_format.data.owned_sel;
 
@@ -376,8 +376,8 @@ void TupleDataCollection::ListWithinListComputeHeapSizes(Vector &heap_sizes_v, c
 	for (idx_t i = 0; i < child_list_child_count; i++) {
 		combined_sel.set_index(i, 0);
 	}
-	idx_t combined_list_offset = 0;
 
+	idx_t combined_list_offset = 0;
 	for (idx_t i = 0; i < append_count; i++) {
 		const auto list_idx = list_sel.get_index(append_sel.get_index(i));
 		if (!list_validity.RowIsValid(list_idx)) {

package/src/duckdb/src/common/types/uuid.cpp

@@ -49,7 +49,7 @@ bool UUID::FromString(string str, hugeint_t &result) {
 		count++;
 	}
 	// Flip the first bit to make `order by uuid` same as `order by uuid::varchar`
-	result.upper ^= (int64_t(1) << 63);
+	result.upper ^= (uint64_t(1) << 63);
 	return count == 32;
 }
 
@@ -61,7 +61,7 @@ void UUID::ToString(hugeint_t input, char *buf) {
 	};
 
 	// Flip back before convert to string
-	int64_t upper = input.upper ^ (int64_t(1) << 63);
+	int64_t upper = input.upper ^ (uint64_t(1) << 63);
 	idx_t pos = 0;
 	byte_to_hex(upper >> 56 & 0xFF, buf, pos);
 	byte_to_hex(upper >> 48 & 0xFF, buf, pos);

package/src/duckdb/src/common/types/validity_mask.cpp

@@ -96,6 +96,39 @@ void ValidityMask::SliceInPlace(const ValidityMask &other, idx_t target_offset,
 		memcpy(target_validity + target_offset_entries, source_validity + source_offset_entries,
 		       sizeof(validity_t) * EntryCount(count));
 		return;
+	} else if (IsAligned(target_offset)) {
+		//	Simple common case where we are shifting into an aligned mask (e.g., 0 in Slice above)
+		const idx_t entire_units = count / BITS_PER_VALUE;
+		const idx_t ragged = count % BITS_PER_VALUE;
+		const idx_t tail = source_offset % BITS_PER_VALUE;
+		const idx_t head = BITS_PER_VALUE - tail;
+		auto source_validity = other.GetData() + (source_offset / BITS_PER_VALUE);
+		auto target_validity = this->GetData() + (target_offset / BITS_PER_VALUE);
+		auto src_entry = *source_validity++;
+		for (idx_t i = 0; i < entire_units; ++i) {
+			//	Start with head of previous src
+			validity_t tgt_entry = src_entry >> tail;
+			src_entry = *source_validity++;
+			// 	Add in tail of current src
+			tgt_entry |= (src_entry << head);
+			*target_validity++ = tgt_entry;
+		}
+		//	Finish last ragged entry
+		if (ragged) {
+			//	Start with head of previous src
+			validity_t tgt_entry = (src_entry >> tail);
+			//  Add in the tail of the next src, if head was too small
+			if (head < ragged) {
+				src_entry = *source_validity++;
+				tgt_entry |= (src_entry << head);
+			}
+			//  Mask off the bits that go past the ragged end
+			tgt_entry &= (ValidityBuffer::MAX_ENTRY >> (BITS_PER_VALUE - ragged));
+			//	Restore the ragged end of the target
+			tgt_entry |= *target_validity & (ValidityBuffer::MAX_ENTRY << ragged);
+			*target_validity++ = tgt_entry;
+		}
+		return;
 	}
 
 	// FIXME: use bitwise operations here

package/src/duckdb/src/common/virtual_file_system.cpp

@@ -28,7 +28,7 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
 		}
 	}
 	// open the base file handle
-	auto file_handle = FindFileSystem(path)->OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED, opener);
+	auto file_handle = FindFileSystem(path).OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED, opener);
 	if (file_handle->GetType() == FileType::FILE_TYPE_FIFO) {
 		file_handle = PipeFileSystem::OpenPipe(std::move(file_handle));
 	} else if (compression != FileCompressionType::UNCOMPRESSED) {
@@ -42,4 +42,141 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
 	return file_handle;
 }
 
+void VirtualFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
+	handle.file_system.Read(handle, buffer, nr_bytes, location);
+}
+
+void VirtualFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
+	handle.file_system.Write(handle, buffer, nr_bytes, location);
+}
+
+int64_t VirtualFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
+	return handle.file_system.Read(handle, buffer, nr_bytes);
+}
+
+int64_t VirtualFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) {
+	return handle.file_system.Write(handle, buffer, nr_bytes);
+}
+
+int64_t VirtualFileSystem::GetFileSize(FileHandle &handle) {
+	return handle.file_system.GetFileSize(handle);
+}
+time_t VirtualFileSystem::GetLastModifiedTime(FileHandle &handle) {
+	return handle.file_system.GetLastModifiedTime(handle);
+}
+FileType VirtualFileSystem::GetFileType(FileHandle &handle) {
+	return handle.file_system.GetFileType(handle);
+}
+
+void VirtualFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
+	handle.file_system.Truncate(handle, new_size);
+}
+
+void VirtualFileSystem::FileSync(FileHandle &handle) {
+	handle.file_system.FileSync(handle);
+}
+
+// need to look up correct fs for this
+bool VirtualFileSystem::DirectoryExists(const string &directory) {
+	return FindFileSystem(directory).DirectoryExists(directory);
+}
+void VirtualFileSystem::CreateDirectory(const string &directory) {
+	FindFileSystem(directory).CreateDirectory(directory);
+}
+
+void VirtualFileSystem::RemoveDirectory(const string &directory) {
+	FindFileSystem(directory).RemoveDirectory(directory);
+}
+
+bool VirtualFileSystem::ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
+                                  FileOpener *opener) {
+	return FindFileSystem(directory).ListFiles(directory, callback, opener);
+}
+
+void VirtualFileSystem::MoveFile(const string &source, const string &target) {
+	FindFileSystem(source).MoveFile(source, target);
+}
+
+bool VirtualFileSystem::FileExists(const string &filename) {
+	return FindFileSystem(filename).FileExists(filename);
+}
+
+bool VirtualFileSystem::IsPipe(const string &filename) {
+	return FindFileSystem(filename).IsPipe(filename);
+}
+void VirtualFileSystem::RemoveFile(const string &filename) {
+	FindFileSystem(filename).RemoveFile(filename);
+}
+
+vector<string> VirtualFileSystem::Glob(const string &path, FileOpener *opener) {
+	return FindFileSystem(path).Glob(path, opener);
+}
+
+void VirtualFileSystem::RegisterSubSystem(unique_ptr<FileSystem> fs) {
+	sub_systems.push_back(std::move(fs));
+}
+
+void VirtualFileSystem::UnregisterSubSystem(const string &name) {
+	for (auto sub_system = sub_systems.begin(); sub_system != sub_systems.end(); sub_system++) {
+		if (sub_system->get()->GetName() == name) {
+			sub_systems.erase(sub_system);
+			return;
+		}
+	}
+	throw InvalidInputException("Could not find filesystem with name %s", name);
+}
+
+void VirtualFileSystem::RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) {
+	compressed_fs[compression_type] = std::move(fs);
+}
+
+vector<string> VirtualFileSystem::ListSubSystems() {
+	vector<string> names(sub_systems.size());
+	for (idx_t i = 0; i < sub_systems.size(); i++) {
+		names[i] = sub_systems[i]->GetName();
+	}
+	return names;
+}
+
+std::string VirtualFileSystem::GetName() const {
+	return "VirtualFileSystem";
+}
+
+void VirtualFileSystem::SetDisabledFileSystems(const vector<string> &names) {
+	unordered_set<string> new_disabled_file_systems;
+	for (auto &name : names) {
+		if (name.empty()) {
+			continue;
+		}
+		if (new_disabled_file_systems.find(name) != new_disabled_file_systems.end()) {
+			throw InvalidInputException("Duplicate disabled file system \"%s\"", name);
+		}
+		new_disabled_file_systems.insert(name);
+	}
+	for (auto &disabled_fs : disabled_file_systems) {
+		if (new_disabled_file_systems.find(disabled_fs) == new_disabled_file_systems.end()) {
+			throw InvalidInputException("File system \"%s\" has been disabled previously, it cannot be re-enabled",
+			                            disabled_fs);
+		}
+	}
+	disabled_file_systems = std::move(new_disabled_file_systems);
+}
+
+FileSystem &VirtualFileSystem::FindFileSystem(const string &path) {
+	auto &fs = FindFileSystemInternal(path);
+	if (!disabled_file_systems.empty() && disabled_file_systems.find(fs.GetName()) != disabled_file_systems.end()) {
+		throw PermissionException("File system %s has been disabled by configuration", fs.GetName());
+	}
+	return fs;
+}
+
+FileSystem &VirtualFileSystem::FindFileSystemInternal(const string &path) {
+	for (auto &sub_system : sub_systems) {
+		if (sub_system->CanHandleFile(path)) {
+			return *sub_system;
+		}
+	}
+	return *default_fs;
+}
+
 } // namespace duckdb

package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp

@@ -21,8 +21,10 @@ struct ReservoirQuantileState {
 		if (new_len <= len) {
 			return;
 		}
+		T *old_v = v;
 		v = (T *)realloc(v, new_len * sizeof(T));
 		if (!v) {
+			free(old_v);
 			throw InternalException("Memory allocation failure");
 		}
 		len = new_len;

package/src/duckdb/src/core_functions/aggregate/nested/list.cpp

@@ -63,9 +63,9 @@ struct ListFunction {
 
 	template <class STATE>
 	static void Destroy(STATE &state, AggregateInputData &aggr_input_data) {
-		auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
-		list_bind_data.functions.Destroy(aggr_input_data.allocator, state.linked_list);
+		// nop
 	}
+
 	static bool IgnoreNull() {
 		return false;
 	}

package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp

@@ -37,16 +37,16 @@ struct RegrAvgFunction {
 };
 struct RegrAvgXFunction : RegrAvgFunction {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		state.sum += y;
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		state.sum += x;
 		state.count++;
 	}
 };
 
 struct RegrAvgYFunction : RegrAvgFunction {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		state.sum += x;
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		state.sum += y;
 		state.count++;
 	}
 };

package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp

@@ -23,11 +23,11 @@ struct RegrInterceptOperation {
 	}
 
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
 		state.count++;
-		state.sum_x += y;
-		state.sum_y += x;
-		RegrSlopeOperation::Operation<A_TYPE, B_TYPE, RegrSlopeState, OP>(state.slope, x, y, idata);
+		state.sum_x += x;
+		state.sum_y += y;
+		RegrSlopeOperation::Operation<A_TYPE, B_TYPE, RegrSlopeState, OP>(state.slope, y, x, idata);
 	}
 
 	template <class STATE, class OP>

package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp

@@ -1,3 +1,4 @@
+// REGR_R2(y, x)
 // Returns the coefficient of determination for non-null pairs in a group.
 // It is computed for non-null pairs using the following formula:
 // null                 if var_pop(x) = 0, else
@@ -24,10 +25,10 @@ struct RegrR2Operation {
 	}
 
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		CorrOperation::Operation<A_TYPE, B_TYPE, CorrState, OP>(state.corr, x, y, idata);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_x, y);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_y, x);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		CorrOperation::Operation<A_TYPE, B_TYPE, CorrState, OP>(state.corr, y, x, idata);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_x, x);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_y, y);
 	}
 
 	template <class STATE, class OP>

package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp

@@ -1,6 +1,6 @@
-// regr_sxx
+// REGR_SXX(y, x)
 // Returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs.
-// regrsyy
+// REGR_SYY(y, x)
 // Returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs.
 
 #include "duckdb/core_functions/aggregate/regression/regr_count.hpp"
@@ -48,17 +48,17 @@ struct RegrBaseOperation {
 
 struct RegrSXXOperation : RegrBaseOperation {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, x, y, idata);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, y);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, y, x, idata);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, x);
 	}
 };
 
 struct RegrSYYOperation : RegrBaseOperation {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, x, y, idata);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, x);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, y, x, idata);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, y);
 	}
 };
 

package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp

@@ -1,3 +1,4 @@
+// REGR_SXY(y, x)
 // Returns REGR_COUNT(expr1, expr2) * COVAR_POP(expr1, expr2) for non-null pairs.
 
 #include "duckdb/core_functions/aggregate/regression/regr_count.hpp"
@@ -20,9 +21,9 @@ struct RegrSXYOperation {
 	}
 
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, x, y, idata);
-		CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, x, y, idata);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, y, x, idata);
+		CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, y, x, idata);
 	}
 
 	template <class STATE, class OP>

package/src/duckdb/src/core_functions/function_list.cpp

@@ -137,7 +137,9 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION(EnumRangeFun),
 	DUCKDB_SCALAR_FUNCTION(EnumRangeBoundaryFun),
 	DUCKDB_SCALAR_FUNCTION_SET(EpochFun),
-	DUCKDB_SCALAR_FUNCTION(EpochMsFun),
+	DUCKDB_SCALAR_FUNCTION_SET(EpochMsFun),
+	DUCKDB_SCALAR_FUNCTION_SET(EpochNsFun),
+	DUCKDB_SCALAR_FUNCTION_SET(EpochUsFun),
 	DUCKDB_SCALAR_FUNCTION_SET(EraFun),
 	DUCKDB_SCALAR_FUNCTION(ErrorFun),
 	DUCKDB_SCALAR_FUNCTION(EvenFun),
@@ -211,7 +213,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_AGGREGATE_FUNCTION_SET(MadFun),
 	DUCKDB_SCALAR_FUNCTION_SET(MakeDateFun),
 	DUCKDB_SCALAR_FUNCTION(MakeTimeFun),
-	DUCKDB_SCALAR_FUNCTION(MakeTimestampFun),
+	DUCKDB_SCALAR_FUNCTION_SET(MakeTimestampFun),
 	DUCKDB_SCALAR_FUNCTION(MapFun),
 	DUCKDB_SCALAR_FUNCTION(MapConcatFun),
 	DUCKDB_SCALAR_FUNCTION(MapEntriesFun),