duckdb 0.10.3-dev8.0 → 1.0.1-dev15.0

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -228,12 +228,16 @@ unique_ptr<LocalSinkState> PhysicalCopyToFile::GetLocalSinkState(ExecutionContex
 	return std::move(res);
 }
 
-void CheckDirectory(FileSystem &fs, const string &file_path, bool overwrite) {
-	if (fs.IsRemoteFile(file_path) && overwrite) {
-		// we only remove files for local file systems
-		// as remote file systems (e.g. S3) do not support RemoveFile
+void CheckDirectory(FileSystem &fs, const string &file_path, CopyOverwriteMode overwrite_mode) {
+	if (overwrite_mode == CopyOverwriteMode::COPY_OVERWRITE_OR_IGNORE) {
+		// with overwrite or ignore we fully ignore the presence of any files instead of erasing them
 		return;
 	}
+	if (fs.IsRemoteFile(file_path) && overwrite_mode == CopyOverwriteMode::COPY_OVERWRITE) {
+		// we can only remove files for local file systems currently
+		// as remote file systems (e.g. S3) do not support RemoveFile
+		throw NotImplementedException("OVERWRITE is not supported for remote file systems");
+	}
 	vector<string> file_list;
 	vector<string> directory_list;
 	directory_list.push_back(file_path);
@@ -251,13 +255,12 @@ void CheckDirectory(FileSystem &fs, const string &file_path, bool overwrite) {
 	if (file_list.empty()) {
 		return;
 	}
-	if (overwrite) {
+	if (overwrite_mode == CopyOverwriteMode::COPY_OVERWRITE) {
 		for (auto &file : file_list) {
 			fs.RemoveFile(file);
 		}
 	} else {
-		throw IOException("Directory \"%s\" is not empty! Enable OVERWRITE_OR_IGNORE option to force writing",
-		                  file_path);
+		throw IOException("Directory \"%s\" is not empty! Enable OVERWRITE option to overwrite files", file_path);
 	}
 }
 
@@ -272,11 +275,11 @@ unique_ptr<GlobalSinkState> PhysicalCopyToFile::GetGlobalSinkState(ClientContext
 				throw IOException("Cannot write to \"%s\" - it exists and is a file, not a directory!", file_path);
 			} else {
 				// for local files we can remove the file if OVERWRITE_OR_IGNORE is enabled
-				if (overwrite_or_ignore) {
+				if (overwrite_mode == CopyOverwriteMode::COPY_OVERWRITE) {
 					fs.RemoveFile(file_path);
 				} else {
 					throw IOException("Cannot write to \"%s\" - it exists and is a file, not a directory! Enable "
-					                  "OVERWRITE_OR_IGNORE option to force writing",
+					                  "OVERWRITE option to overwrite the file",
 					                  file_path);
 				}
 			}
@@ -285,7 +288,7 @@ unique_ptr<GlobalSinkState> PhysicalCopyToFile::GetGlobalSinkState(ClientContext
 		if (!fs.DirectoryExists(file_path)) {
 			fs.CreateDirectory(file_path);
 		} else {
-			CheckDirectory(fs, file_path, overwrite_or_ignore);
+			CheckDirectory(fs, file_path, overwrite_mode);
 		}
 
 		auto state = make_uniq<CopyToFunctionGlobalState>(nullptr);

package/src/duckdb/src/execution/physical_plan/plan_copy_to_file.cpp

@@ -17,7 +17,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCopyToFile
 		op.file_path = fs.JoinPath(path, "tmp_" + base);
 	}
 	if (op.per_thread_output || op.file_size_bytes.IsValid() || op.partition_output || !op.partition_columns.empty() ||
-	    op.overwrite_or_ignore) {
+	    op.overwrite_mode != CopyOverwriteMode::COPY_ERROR_ON_CONFLICT) {
 		// hive-partitioning/per-thread output does not care about insertion order, and does not support batch indexes
 		preserve_insertion_order = false;
 		supports_batch_index = false;
@@ -42,7 +42,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCopyToFile
 	auto copy = make_uniq<PhysicalCopyToFile>(op.types, op.function, std::move(op.bind_data), op.estimated_cardinality);
 	copy->file_path = op.file_path;
 	copy->use_tmp_file = op.use_tmp_file;
-	copy->overwrite_or_ignore = op.overwrite_or_ignore;
+	copy->overwrite_mode = op.overwrite_mode;
 	copy->filename_pattern = op.filename_pattern;
 	copy->file_extension = op.file_extension;
 	copy->per_thread_output = op.per_thread_output;

package/src/duckdb/src/function/table/system/duckdb_extensions.cpp

@@ -125,7 +125,7 @@ unique_ptr<GlobalTableFunctionState> DuckDBExtensionsInit(ClientContext &context
 		if (entry == installed_extensions.end()) {
 			installed_extensions[info.name] = std::move(info);
 		} else {
-			if (!entry->second.loaded) {
+			if (entry->second.install_mode != ExtensionInstallMode::STATICALLY_LINKED) {
 				entry->second.file_path = info.file_path;
 				entry->second.install_mode = info.install_mode;
 				entry->second.installed_from = info.installed_from;
@@ -144,13 +144,12 @@ unique_ptr<GlobalTableFunctionState> DuckDBExtensionsInit(ClientContext &context
 		auto &ext_info = e.second;
 		auto entry = installed_extensions.find(ext_name);
 		if (entry == installed_extensions.end() || !entry->second.installed) {
-			ExtensionInformation info;
+			ExtensionInformation &info = installed_extensions[ext_name];
 			info.name = ext_name;
 			info.loaded = true;
 			info.extension_version = ext_info.version;
 			info.installed = ext_info.mode == ExtensionInstallMode::STATICALLY_LINKED;
 			info.install_mode = ext_info.mode;
-			installed_extensions[ext_name] = std::move(info);
 		} else {
 			entry->second.loaded = true;
 			entry->second.extension_version = ext_info.version;

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,17 +1,17 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "3"
+#define DUCKDB_PATCH_VERSION "0"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
-#define DUCKDB_MINOR_VERSION 10
+#define DUCKDB_MINOR_VERSION 0
 #endif
 #ifndef DUCKDB_MAJOR_VERSION
-#define DUCKDB_MAJOR_VERSION 0
+#define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v0.10.3"
+#define DUCKDB_VERSION "v1.0.0"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "70fd6a8a24"
+#define DUCKDB_SOURCE_ID "1f98600c2c"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/enum_util.hpp

@@ -98,6 +98,8 @@ enum class ConflictManagerMode : uint8_t;
 
 enum class ConstraintType : uint8_t;
 
+enum class CopyOverwriteMode : uint8_t;
+
 enum class DataFileType : uint8_t;
 
 enum class DatePartSpecifier : uint8_t;
@@ -434,6 +436,9 @@ const char* EnumUtil::ToChars<ConflictManagerMode>(ConflictManagerMode value);
 template<>
 const char* EnumUtil::ToChars<ConstraintType>(ConstraintType value);
 
+template<>
+const char* EnumUtil::ToChars<CopyOverwriteMode>(CopyOverwriteMode value);
+
 template<>
 const char* EnumUtil::ToChars<DataFileType>(DataFileType value);
 
@@ -888,6 +893,9 @@ ConflictManagerMode EnumUtil::FromString<ConflictManagerMode>(const char *value)
 template<>
 ConstraintType EnumUtil::FromString<ConstraintType>(const char *value);
 
+template<>
+CopyOverwriteMode EnumUtil::FromString<CopyOverwriteMode>(const char *value);
+
 template<>
 DataFileType EnumUtil::FromString<DataFileType>(const char *value);
 

package/src/duckdb/src/include/duckdb/common/enums/copy_overwrite_mode.hpp

@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/enums/copy_overwrite_mode.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/constants.hpp"
+#include "duckdb/common/vector.hpp"
+
+namespace duckdb {
+
+enum class CopyOverwriteMode : uint8_t { COPY_ERROR_ON_CONFLICT = 0, COPY_OVERWRITE = 1, COPY_OVERWRITE_OR_IGNORE = 2 };
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp

@@ -52,6 +52,7 @@ struct MultiFileReaderBindData {
 struct MultiFileReaderGlobalState {
 	MultiFileReaderGlobalState(vector<LogicalType> extra_columns_p, optional_ptr<const MultiFileList> file_list_p)
 	    : extra_columns(std::move(extra_columns_p)), file_list(file_list_p) {};
+	virtual ~MultiFileReaderGlobalState();
 
 	//! extra columns that will be produced during scanning
 	const vector<LogicalType> extra_columns;

package/src/duckdb/src/include/duckdb/common/shared_ptr.hpp

@@ -37,9 +37,9 @@ struct compatible_with_t : std::is_convertible<U *, T *> {}; // NOLINT: invalid
 
 } // namespace duckdb
 
-#include "duckdb/common/shared_ptr.ipp"
-#include "duckdb/common/weak_ptr.ipp"
-#include "duckdb/common/enable_shared_from_this.ipp"
+#include "duckdb/common/shared_ptr_ipp.hpp"
+#include "duckdb/common/weak_ptr_ipp.hpp"
+#include "duckdb/common/enable_shared_from_this_ipp.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/function/copy_function.hpp"
 #include "duckdb/parser/parsed_data/copy_info.hpp"
+#include "duckdb/common/enums/copy_overwrite_mode.hpp"
 
 namespace duckdb {
 
@@ -31,7 +32,7 @@ public:
 	bool use_tmp_file;
 	FilenamePattern filename_pattern;
 	string file_extension;
-	bool overwrite_or_ignore;
+	CopyOverwriteMode overwrite_mode;
 	bool parallel;
 	bool per_thread_output;
 	optional_idx file_size_bytes;

package/src/duckdb/src/include/duckdb/main/attached_database.hpp

@@ -54,6 +54,11 @@ public:
 	DatabaseInstance &GetDatabase() {
 		return db;
 	}
+
+	optional_ptr<StorageExtension> GetStorageExtension() {
+		return storage_extension;
+	}
+
 	const string &GetName() const {
 		return name;
 	}
@@ -74,6 +79,7 @@ private:
 	unique_ptr<TransactionManager> transaction_manager;
 	AttachedDatabaseType type;
 	optional_ptr<Catalog> parent_catalog;
+	optional_ptr<StorageExtension> storage_extension;
 	bool is_initial_database = false;
 	bool is_closed = false;
 };

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -208,6 +208,10 @@ struct DBConfigOptions {
 	bool allow_extensions_metadata_mismatch = false;
 	//! Enable emitting FSST Vectors
 	bool enable_fsst_vectors = false;
+	//! Enable VIEWs to create dependencies
+	bool enable_view_dependencies = false;
+	//! Enable macros to create dependencies
+	bool enable_macro_dependencies = false;
 	//! Start transactions immediately in all attached databases - instead of lazily when a database is referenced
 	bool immediate_transaction_mode = false;
 	//! Debug setting - how to initialize  blocks in the storage layer when allocating

package/src/duckdb/src/include/duckdb/main/extension_helper.hpp

@@ -88,10 +88,12 @@ public:
 	static unique_ptr<ExtensionInstallInfo> InstallExtension(ClientContext &context, const string &extension,
 	                                                         bool force_install,
 	                                                         optional_ptr<ExtensionRepository> repository = nullptr,
+	                                                         bool throw_on_origin_mismatch = false,
 	                                                         const string &version = "");
 	static unique_ptr<ExtensionInstallInfo> InstallExtension(DBConfig &config, FileSystem &fs, const string &extension,
 	                                                         bool force_install,
 	                                                         optional_ptr<ExtensionRepository> repository = nullptr,
+	                                                         bool throw_on_origin_mismatch = false,
 	                                                         const string &version = "");
 	//! Load an extension
 	static void LoadExternalExtension(ClientContext &context, const string &extension);
@@ -213,12 +215,10 @@ public:
 	static bool CreateSuggestions(const string &extension_name, string &message);
 
 private:
-	static unique_ptr<ExtensionInstallInfo> InstallExtensionInternal(DBConfig &config, FileSystem &fs,
-	                                                                 const string &local_path, const string &extension,
-	                                                                 bool force_install, const string &version,
-	                                                                 optional_ptr<ExtensionRepository> repository,
-	                                                                 optional_ptr<HTTPLogger> http_logger = nullptr,
-	                                                                 optional_ptr<ClientContext> context = nullptr);
+	static unique_ptr<ExtensionInstallInfo> InstallExtensionInternal(
+	    DBConfig &config, FileSystem &fs, const string &local_path, const string &extension, bool force_install,
+	    bool throw_on_origin_mismatch, const string &version, optional_ptr<ExtensionRepository> repository,
+	    optional_ptr<HTTPLogger> http_logger = nullptr, optional_ptr<ClientContext> context = nullptr);
 	static const vector<string> PathComponents();
 	static string DefaultExtensionFolder(FileSystem &fs);
 	static bool AllowAutoInstall(const string &extension);

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -206,6 +206,26 @@ struct EnableExternalAccessSetting {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct EnableMacrosDependencies {
+	static constexpr const char *Name = "enable_macro_dependencies";
+	static constexpr const char *Description =
+	    "Enable created MACROs to create dependencies on the referenced objects (such as tables)";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct EnableViewDependencies {
+	static constexpr const char *Name = "enable_view_dependencies";
+	static constexpr const char *Description =
+	    "Enable created VIEWs to create dependencies on the referenced objects (such as tables)";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct EnableFSSTVectors {
 	static constexpr const char *Name = "enable_fsst_vectors";
 	static constexpr const char *Description =

package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp

@@ -51,16 +51,16 @@ public:
 	                                              vector<JoinCondition> conditions,
 	                                              vector<unique_ptr<Expression>> arbitrary_expressions);
 
-	static void ExtractJoinConditions(ClientContext &context, JoinType type, unique_ptr<LogicalOperator> &left_child,
-	                                  unique_ptr<LogicalOperator> &right_child, unique_ptr<Expression> condition,
-	                                  vector<JoinCondition> &conditions,
+	static void ExtractJoinConditions(ClientContext &context, JoinType type, JoinRefType ref_type,
+	                                  unique_ptr<LogicalOperator> &left_child, unique_ptr<LogicalOperator> &right_child,
+	                                  unique_ptr<Expression> condition, vector<JoinCondition> &conditions,
 	                                  vector<unique_ptr<Expression>> &arbitrary_expressions);
-	static void ExtractJoinConditions(ClientContext &context, JoinType type, unique_ptr<LogicalOperator> &left_child,
-	                                  unique_ptr<LogicalOperator> &right_child,
+	static void ExtractJoinConditions(ClientContext &context, JoinType type, JoinRefType ref_type,
+	                                  unique_ptr<LogicalOperator> &left_child, unique_ptr<LogicalOperator> &right_child,
 	                                  vector<unique_ptr<Expression>> &expressions, vector<JoinCondition> &conditions,
 	                                  vector<unique_ptr<Expression>> &arbitrary_expressions);
-	static void ExtractJoinConditions(ClientContext &context, JoinType type, unique_ptr<LogicalOperator> &left_child,
-	                                  unique_ptr<LogicalOperator> &right_child,
+	static void ExtractJoinConditions(ClientContext &context, JoinType type, JoinRefType ref_type,
+	                                  unique_ptr<LogicalOperator> &left_child, unique_ptr<LogicalOperator> &right_child,
 	                                  const unordered_set<idx_t> &left_bindings,
 	                                  const unordered_set<idx_t> &right_bindings,
 	                                  vector<unique_ptr<Expression>> &expressions, vector<JoinCondition> &conditions,

package/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/common/optional_idx.hpp"
 #include "duckdb/function/copy_function.hpp"
 #include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/common/enums/copy_overwrite_mode.hpp"
 
 namespace duckdb {
 
@@ -33,7 +34,7 @@ public:
 	bool use_tmp_file;
 	FilenamePattern filename_pattern;
 	string file_extension;
-	bool overwrite_or_ignore;
+	CopyOverwriteMode overwrite_mode;
 	bool per_thread_output;
 	optional_idx file_size_bytes;
 

package/src/duckdb/src/include/duckdb/storage/storage_extension.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/enums/access_mode.hpp"
 #include "duckdb/parser/tableref/table_function_ref.hpp"
+#include "duckdb/storage/storage_manager.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -40,6 +41,12 @@ public:
 
 	virtual ~StorageExtension() {
 	}
+
+	virtual void OnCheckpointStart(AttachedDatabase &db, CheckpointOptions checkpoint_options) {
+	}
+
+	virtual void OnCheckpointEnd(AttachedDatabase &db, CheckpointOptions checkpoint_options) {
+	}
 };
 
 } // namespace duckdb