duckdb 0.8.2-dev150.0 → 0.8.2-dev1549.0

package/src/duckdb/src/common/virtual_file_system.cpp

@@ -28,7 +28,7 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
 		}
 	}
 	// open the base file handle
-	auto file_handle = FindFileSystem(path)->OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED, opener);
+	auto file_handle = FindFileSystem(path).OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED, opener);
 	if (file_handle->GetType() == FileType::FILE_TYPE_FIFO) {
 		file_handle = PipeFileSystem::OpenPipe(std::move(file_handle));
 	} else if (compression != FileCompressionType::UNCOMPRESSED) {
@@ -42,4 +42,141 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
 	return file_handle;
 }
 
+void VirtualFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
+	handle.file_system.Read(handle, buffer, nr_bytes, location);
+}
+
+void VirtualFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
+	handle.file_system.Write(handle, buffer, nr_bytes, location);
+}
+
+int64_t VirtualFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
+	return handle.file_system.Read(handle, buffer, nr_bytes);
+}
+
+int64_t VirtualFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) {
+	return handle.file_system.Write(handle, buffer, nr_bytes);
+}
+
+int64_t VirtualFileSystem::GetFileSize(FileHandle &handle) {
+	return handle.file_system.GetFileSize(handle);
+}
+time_t VirtualFileSystem::GetLastModifiedTime(FileHandle &handle) {
+	return handle.file_system.GetLastModifiedTime(handle);
+}
+FileType VirtualFileSystem::GetFileType(FileHandle &handle) {
+	return handle.file_system.GetFileType(handle);
+}
+
+void VirtualFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
+	handle.file_system.Truncate(handle, new_size);
+}
+
+void VirtualFileSystem::FileSync(FileHandle &handle) {
+	handle.file_system.FileSync(handle);
+}
+
+// need to look up correct fs for this
+bool VirtualFileSystem::DirectoryExists(const string &directory) {
+	return FindFileSystem(directory).DirectoryExists(directory);
+}
+void VirtualFileSystem::CreateDirectory(const string &directory) {
+	FindFileSystem(directory).CreateDirectory(directory);
+}
+
+void VirtualFileSystem::RemoveDirectory(const string &directory) {
+	FindFileSystem(directory).RemoveDirectory(directory);
+}
+
+bool VirtualFileSystem::ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
+                                  FileOpener *opener) {
+	return FindFileSystem(directory).ListFiles(directory, callback, opener);
+}
+
+void VirtualFileSystem::MoveFile(const string &source, const string &target) {
+	FindFileSystem(source).MoveFile(source, target);
+}
+
+bool VirtualFileSystem::FileExists(const string &filename) {
+	return FindFileSystem(filename).FileExists(filename);
+}
+
+bool VirtualFileSystem::IsPipe(const string &filename) {
+	return FindFileSystem(filename).IsPipe(filename);
+}
+void VirtualFileSystem::RemoveFile(const string &filename) {
+	FindFileSystem(filename).RemoveFile(filename);
+}
+
+vector<string> VirtualFileSystem::Glob(const string &path, FileOpener *opener) {
+	return FindFileSystem(path).Glob(path, opener);
+}
+
+void VirtualFileSystem::RegisterSubSystem(unique_ptr<FileSystem> fs) {
+	sub_systems.push_back(std::move(fs));
+}
+
+void VirtualFileSystem::UnregisterSubSystem(const string &name) {
+	for (auto sub_system = sub_systems.begin(); sub_system != sub_systems.end(); sub_system++) {
+		if (sub_system->get()->GetName() == name) {
+			sub_systems.erase(sub_system);
+			return;
+		}
+	}
+	throw InvalidInputException("Could not find filesystem with name %s", name);
+}
+
+void VirtualFileSystem::RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) {
+	compressed_fs[compression_type] = std::move(fs);
+}
+
+vector<string> VirtualFileSystem::ListSubSystems() {
+	vector<string> names(sub_systems.size());
+	for (idx_t i = 0; i < sub_systems.size(); i++) {
+		names[i] = sub_systems[i]->GetName();
+	}
+	return names;
+}
+
+std::string VirtualFileSystem::GetName() const {
+	return "VirtualFileSystem";
+}
+
+void VirtualFileSystem::SetDisabledFileSystems(const vector<string> &names) {
+	unordered_set<string> new_disabled_file_systems;
+	for (auto &name : names) {
+		if (name.empty()) {
+			continue;
+		}
+		if (new_disabled_file_systems.find(name) != new_disabled_file_systems.end()) {
+			throw InvalidInputException("Duplicate disabled file system \"%s\"", name);
+		}
+		new_disabled_file_systems.insert(name);
+	}
+	for (auto &disabled_fs : disabled_file_systems) {
+		if (new_disabled_file_systems.find(disabled_fs) == new_disabled_file_systems.end()) {
+			throw InvalidInputException("File system \"%s\" has been disabled previously, it cannot be re-enabled",
+			                            disabled_fs);
+		}
+	}
+	disabled_file_systems = std::move(new_disabled_file_systems);
+}
+
+FileSystem &VirtualFileSystem::FindFileSystem(const string &path) {
+	auto &fs = FindFileSystemInternal(path);
+	if (!disabled_file_systems.empty() && disabled_file_systems.find(fs.GetName()) != disabled_file_systems.end()) {
+		throw PermissionException("File system %s has been disabled by configuration", fs.GetName());
+	}
+	return fs;
+}
+
+FileSystem &VirtualFileSystem::FindFileSystemInternal(const string &path) {
+	for (auto &sub_system : sub_systems) {
+		if (sub_system->CanHandleFile(path)) {
+			return *sub_system;
+		}
+	}
+	return *default_fs;
+}
+
 } // namespace duckdb

package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp

@@ -21,8 +21,10 @@ struct ReservoirQuantileState {
 		if (new_len <= len) {
 			return;
 		}
+		T *old_v = v;
 		v = (T *)realloc(v, new_len * sizeof(T));
 		if (!v) {
+			free(old_v);
 			throw InternalException("Memory allocation failure");
 		}
 		len = new_len;

package/src/duckdb/src/core_functions/aggregate/nested/list.cpp

@@ -63,9 +63,9 @@ struct ListFunction {
 
 	template <class STATE>
 	static void Destroy(STATE &state, AggregateInputData &aggr_input_data) {
-		auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
-		list_bind_data.functions.Destroy(aggr_input_data.allocator, state.linked_list);
+		// nop
 	}
+
 	static bool IgnoreNull() {
 		return false;
 	}

package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp

@@ -37,16 +37,16 @@ struct RegrAvgFunction {
 };
 struct RegrAvgXFunction : RegrAvgFunction {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		state.sum += y;
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		state.sum += x;
 		state.count++;
 	}
 };
 
 struct RegrAvgYFunction : RegrAvgFunction {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		state.sum += x;
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		state.sum += y;
 		state.count++;
 	}
 };

package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp

@@ -23,11 +23,11 @@ struct RegrInterceptOperation {
 	}
 
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
 		state.count++;
-		state.sum_x += y;
-		state.sum_y += x;
-		RegrSlopeOperation::Operation<A_TYPE, B_TYPE, RegrSlopeState, OP>(state.slope, x, y, idata);
+		state.sum_x += x;
+		state.sum_y += y;
+		RegrSlopeOperation::Operation<A_TYPE, B_TYPE, RegrSlopeState, OP>(state.slope, y, x, idata);
 	}
 
 	template <class STATE, class OP>

package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp

@@ -1,3 +1,4 @@
+// REGR_R2(y, x)
 // Returns the coefficient of determination for non-null pairs in a group.
 // It is computed for non-null pairs using the following formula:
 // null                 if var_pop(x) = 0, else
@@ -24,10 +25,10 @@ struct RegrR2Operation {
 	}
 
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		CorrOperation::Operation<A_TYPE, B_TYPE, CorrState, OP>(state.corr, x, y, idata);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_x, y);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_y, x);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		CorrOperation::Operation<A_TYPE, B_TYPE, CorrState, OP>(state.corr, y, x, idata);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_x, x);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_y, y);
 	}
 
 	template <class STATE, class OP>

package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp

@@ -1,6 +1,6 @@
-// regr_sxx
+// REGR_SXX(y, x)
 // Returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs.
-// regrsyy
+// REGR_SYY(y, x)
 // Returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs.
 
 #include "duckdb/core_functions/aggregate/regression/regr_count.hpp"
@@ -48,17 +48,17 @@ struct RegrBaseOperation {
 
 struct RegrSXXOperation : RegrBaseOperation {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, x, y, idata);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, y);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, y, x, idata);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, x);
 	}
 };
 
 struct RegrSYYOperation : RegrBaseOperation {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, x, y, idata);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, x);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, y, x, idata);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, y);
 	}
 };
 

package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp

@@ -1,3 +1,4 @@
+// REGR_SXY(y, x)
 // Returns REGR_COUNT(expr1, expr2) * COVAR_POP(expr1, expr2) for non-null pairs.
 
 #include "duckdb/core_functions/aggregate/regression/regr_count.hpp"
@@ -20,9 +21,9 @@ struct RegrSXYOperation {
 	}
 
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, x, y, idata);
-		CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, x, y, idata);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, y, x, idata);
+		CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, y, x, idata);
 	}
 
 	template <class STATE, class OP>

package/src/duckdb/src/core_functions/function_list.cpp

@@ -137,7 +137,9 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION(EnumRangeFun),
 	DUCKDB_SCALAR_FUNCTION(EnumRangeBoundaryFun),
 	DUCKDB_SCALAR_FUNCTION_SET(EpochFun),
-	DUCKDB_SCALAR_FUNCTION(EpochMsFun),
+	DUCKDB_SCALAR_FUNCTION_SET(EpochMsFun),
+	DUCKDB_SCALAR_FUNCTION_SET(EpochNsFun),
+	DUCKDB_SCALAR_FUNCTION_SET(EpochUsFun),
 	DUCKDB_SCALAR_FUNCTION_SET(EraFun),
 	DUCKDB_SCALAR_FUNCTION(ErrorFun),
 	DUCKDB_SCALAR_FUNCTION(EvenFun),
@@ -211,7 +213,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_AGGREGATE_FUNCTION_SET(MadFun),
 	DUCKDB_SCALAR_FUNCTION_SET(MakeDateFun),
 	DUCKDB_SCALAR_FUNCTION(MakeTimeFun),
-	DUCKDB_SCALAR_FUNCTION(MakeTimestampFun),
+	DUCKDB_SCALAR_FUNCTION_SET(MakeTimestampFun),
 	DUCKDB_SCALAR_FUNCTION(MapFun),
 	DUCKDB_SCALAR_FUNCTION(MapConcatFun),
 	DUCKDB_SCALAR_FUNCTION(MapEntriesFun),