duckdb 0.10.3-dev8.0 → 1.0.1-dev15.0

package/src/duckdb/src/main/extension/extension_install.cpp

@@ -136,18 +136,21 @@ bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &me
 unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtension(DBConfig &config, FileSystem &fs,
                                                                    const string &extension, bool force_install,
                                                                    optional_ptr<ExtensionRepository> repository,
+                                                                   bool throw_on_origin_mismatch,
                                                                    const string &version) {
 #ifdef WASM_LOADABLE_EXTENSIONS
 	// Install is currently a no-op
 	return nullptr;
 #endif
 	string local_path = ExtensionDirectory(config, fs);
-	return InstallExtensionInternal(config, fs, local_path, extension, force_install, version, repository);
+	return InstallExtensionInternal(config, fs, local_path, extension, force_install, throw_on_origin_mismatch, version,
+	                                repository);
 }
 
 unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtension(ClientContext &context, const string &extension,
                                                                    bool force_install,
                                                                    optional_ptr<ExtensionRepository> repository,
+                                                                   bool throw_on_origin_mismatch,
                                                                    const string &version) {
 #ifdef WASM_LOADABLE_EXTENSIONS
 	// Install is currently a no-op
@@ -158,8 +161,8 @@ unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtension(ClientContext
 	string local_path = ExtensionDirectory(context);
 	optional_ptr<HTTPLogger> http_logger =
 	    ClientConfig::GetConfig(context).enable_http_logging ? context.client_data->http_logger.get() : nullptr;
-	return InstallExtensionInternal(db_config, fs, local_path, extension, force_install, version, repository,
-	                                http_logger, context);
+	return InstallExtensionInternal(db_config, fs, local_path, extension, force_install, throw_on_origin_mismatch,
+	                                version, repository, http_logger, context);
 }
 
 unsafe_unique_array<data_t> ReadExtensionFileFromDisk(FileSystem &fs, const string &path, idx_t &file_size) {
@@ -445,8 +448,8 @@ static void ThrowErrorOnMismatchingExtensionOrigin(FileSystem &fs, const string
 
 unique_ptr<ExtensionInstallInfo>
 ExtensionHelper::InstallExtensionInternal(DBConfig &config, FileSystem &fs, const string &local_path,
-                                          const string &extension, bool force_install, const string &version,
-                                          optional_ptr<ExtensionRepository> repository,
+                                          const string &extension, bool force_install, bool throw_on_origin_mismatch,
+                                          const string &version, optional_ptr<ExtensionRepository> repository,
                                           optional_ptr<HTTPLogger> http_logger, optional_ptr<ClientContext> context) {
 #ifdef DUCKDB_DISABLE_EXTENSION_LOAD
 	throw PermissionException("Installing external extensions is disabled through a compile time flag");
@@ -461,7 +464,8 @@ ExtensionHelper::InstallExtensionInternal(DBConfig &config, FileSystem &fs, cons
 
 	if (fs.FileExists(local_extension_path) && !force_install) {
 		// File exists: throw error if origin mismatches
-		if (!config.options.allow_extensions_metadata_mismatch && fs.FileExists(local_extension_path + ".info")) {
+		if (throw_on_origin_mismatch && !config.options.allow_extensions_metadata_mismatch &&
+		    fs.FileExists(local_extension_path + ".info")) {
 			ThrowErrorOnMismatchingExtensionOrigin(fs, local_extension_path, extension_name, extension, repository);
 		}
 

package/src/duckdb/src/main/settings/settings.cpp

@@ -437,6 +437,38 @@ Value EnableExternalAccessSetting::GetSetting(const ClientContext &context) {
 	return Value::BOOLEAN(config.options.enable_external_access);
 }
 
+//===--------------------------------------------------------------------===//
+// Enable Macro Dependencies
+//===--------------------------------------------------------------------===//
+void EnableMacrosDependencies::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	config.options.enable_macro_dependencies = input.GetValue<bool>();
+}
+
+void EnableMacrosDependencies::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.enable_macro_dependencies = DBConfig().options.enable_macro_dependencies;
+}
+
+Value EnableMacrosDependencies::GetSetting(const ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	return Value::BOOLEAN(config.options.enable_macro_dependencies);
+}
+
+//===--------------------------------------------------------------------===//
+// Enable View Dependencies
+//===--------------------------------------------------------------------===//
+void EnableViewDependencies::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	config.options.enable_view_dependencies = input.GetValue<bool>();
+}
+
+void EnableViewDependencies::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.enable_view_dependencies = DBConfig().options.enable_view_dependencies;
+}
+
+Value EnableViewDependencies::GetSetting(const ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	return Value::BOOLEAN(config.options.enable_view_dependencies);
+}
+
 //===--------------------------------------------------------------------===//
 // Enable FSST Vectors
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp

@@ -52,9 +52,9 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownCrossProduct(unique_ptr<Logi
 		vector<JoinCondition> conditions;
 		vector<unique_ptr<Expression>> arbitrary_expressions;
 		const auto join_type = JoinType::INNER;
-		LogicalComparisonJoin::ExtractJoinConditions(GetContext(), join_type, op->children[0], op->children[1],
-		                                             left_bindings, right_bindings, join_expressions, conditions,
-		                                             arbitrary_expressions);
+		LogicalComparisonJoin::ExtractJoinConditions(GetContext(), join_type, join_ref_type, op->children[0],
+		                                             op->children[1], left_bindings, right_bindings, join_expressions,
+		                                             conditions, arbitrary_expressions);
 		// create the join from the join conditions
 		return LogicalComparisonJoin::CreateJoin(GetContext(), join_type, join_ref_type, std::move(op->children[0]),
 		                                         std::move(op->children[1]), std::move(conditions),

package/src/duckdb/src/optimizer/rule/move_constants.cpp

@@ -41,6 +41,10 @@ unique_ptr<Expression> MoveConstantsRule::Apply(LogicalOperator &op, vector<refe
 	D_ASSERT(arithmetic.return_type.IsIntegral());
 	D_ASSERT(arithmetic.children[0]->return_type.IsIntegral());
 	if (inner_constant.value.IsNull() || outer_constant.value.IsNull()) {
+		if (comparison.type == ExpressionType::COMPARE_DISTINCT_FROM ||
+		    comparison.type == ExpressionType::COMPARE_NOT_DISTINCT_FROM) {
+			return nullptr;
+		}
 		return make_uniq<BoundConstantExpression>(Value(comparison.return_type));
 	}
 	auto &constant_type = outer_constant.return_type;

package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp

@@ -417,8 +417,8 @@ unique_ptr<LogicalOperator> Binder::PlanLateralJoin(unique_ptr<LogicalOperator>
 	vector<unique_ptr<Expression>> arbitrary_expressions;
 	if (condition) {
 		// extract join conditions, if there are any
-		LogicalComparisonJoin::ExtractJoinConditions(context, join_type, left, right, std::move(condition), conditions,
-		                                             arbitrary_expressions);
+		LogicalComparisonJoin::ExtractJoinConditions(context, join_type, JoinRefType::REGULAR, left, right,
+		                                             std::move(condition), conditions, arbitrary_expressions);
 	}
 
 	auto perform_delim = PerformDuplicateElimination(*this, correlated);

package/src/duckdb/src/planner/binder/statement/bind_copy.cpp

@@ -55,12 +55,13 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 	}
 
 	bool use_tmp_file = true;
-	bool overwrite_or_ignore = false;
+	CopyOverwriteMode overwrite_mode = CopyOverwriteMode::COPY_ERROR_ON_CONFLICT;
 	FilenamePattern filename_pattern;
 	bool user_set_use_tmp_file = false;
 	bool per_thread_output = false;
 	optional_idx file_size_bytes;
 	vector<idx_t> partition_cols;
+	bool seen_overwrite_mode = false;
 
 	CopyFunctionBindInput bind_input(*stmt.info);
 
@@ -74,8 +75,20 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 		if (loption == "use_tmp_file") {
 			use_tmp_file = GetBooleanArg(context, option.second);
 			user_set_use_tmp_file = true;
-		} else if (loption == "overwrite_or_ignore") {
-			overwrite_or_ignore = GetBooleanArg(context, option.second);
+		} else if (loption == "overwrite_or_ignore" || loption == "overwrite") {
+			if (seen_overwrite_mode) {
+				throw BinderException("Can only set one of OVERWRITE_OR_IGNORE or OVERWRITE");
+			}
+			seen_overwrite_mode = true;
+
+			auto boolean = GetBooleanArg(context, option.second);
+			if (boolean) {
+				if (loption == "overwrite_or_ignore") {
+					overwrite_mode = CopyOverwriteMode::COPY_OVERWRITE_OR_IGNORE;
+				} else if (loption == "overwrite") {
+					overwrite_mode = CopyOverwriteMode::COPY_OVERWRITE;
+				}
+			}
 		} else if (loption == "filename_pattern") {
 			if (option.second.empty()) {
 				throw IOException("FILENAME_PATTERN cannot be empty");
@@ -146,7 +159,7 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 	auto copy = make_uniq<LogicalCopyToFile>(copy_function.function, std::move(function_data), std::move(stmt.info));
 	copy->file_path = file_path;
 	copy->use_tmp_file = use_tmp_file;
-	copy->overwrite_or_ignore = overwrite_or_ignore;
+	copy->overwrite_mode = overwrite_mode;
 	copy->filename_pattern = filename_pattern;
 	copy->file_extension = bind_input.file_extension;
 	copy->per_thread_output = per_thread_output;

package/src/duckdb/src/planner/binder/statement/bind_create.cpp

@@ -145,13 +145,19 @@ void Binder::BindCreateViewInfo(CreateViewInfo &base) {
 	auto view_binder = Binder::CreateBinder(context);
 	auto &dependencies = base.dependencies;
 	auto &catalog = Catalog::GetCatalog(context, base.catalog);
-	view_binder->SetCatalogLookupCallback([&dependencies, &catalog](CatalogEntry &entry) {
-		if (&catalog != &entry.ParentCatalog()) {
-			// Don't register dependencies between catalogs
-			return;
-		}
-		dependencies.AddDependency(entry);
-	});
+
+	auto &db_config = DBConfig::GetConfig(context);
+	auto should_create_dependencies = db_config.options.enable_view_dependencies;
+
+	if (should_create_dependencies) {
+		view_binder->SetCatalogLookupCallback([&dependencies, &catalog](CatalogEntry &entry) {
+			if (&catalog != &entry.ParentCatalog()) {
+				// Don't register dependencies between catalogs
+				return;
+			}
+			dependencies.AddDependency(entry);
+		});
+	}
 	view_binder->can_contain_nulls = true;
 
 	auto copy = base.query->Copy();
@@ -204,14 +210,19 @@ SchemaCatalogEntry &Binder::BindCreateFunctionInfo(CreateInfo &info) {
 	SelectBinder binder(*this, context, sel_node, group_info);
 	auto &dependencies = base.dependencies;
 	auto &catalog = Catalog::GetCatalog(context, info.catalog);
-	binder.SetCatalogLookupCallback([&dependencies, &catalog](CatalogEntry &entry) {
-		if (&catalog != &entry.ParentCatalog()) {
-			// Don't register any cross-catalog dependencies
-			return;
-		}
-		// Register any catalog entry required to bind the macro function
-		dependencies.AddDependency(entry);
-	});
+	auto &db_config = DBConfig::GetConfig(context);
+	auto should_create_dependencies = db_config.options.enable_macro_dependencies;
+
+	if (should_create_dependencies) {
+		binder.SetCatalogLookupCallback([&dependencies, &catalog](CatalogEntry &entry) {
+			if (&catalog != &entry.ParentCatalog()) {
+				// Don't register any cross-catalog dependencies
+				return;
+			}
+			// Register any catalog entry required to bind the macro function
+			dependencies.AddDependency(entry);
+		});
+	}
 	error = binder.Bind(expression, 0, false);
 	if (error.HasError()) {
 		error.Throw();

package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp

@@ -20,6 +20,26 @@
 
 namespace duckdb {
 
+//! Only use conditions that are valid for the join ref type
+static bool IsJoinTypeCondition(const JoinRefType ref_type, const ExpressionType expr_type) {
+	switch (ref_type) {
+	case JoinRefType::ASOF:
+		switch (expr_type) {
+		case ExpressionType::COMPARE_EQUAL:
+		case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
+		case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+		case ExpressionType::COMPARE_GREATERTHAN:
+		case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+		case ExpressionType::COMPARE_LESSTHAN:
+			return true;
+		default:
+			return false;
+		}
+	default:
+		return true;
+	}
+}
+
 //! Create a JoinCondition from a comparison
 static bool CreateJoinCondition(Expression &expr, const unordered_set<idx_t> &left_bindings,
                                 const unordered_set<idx_t> &right_bindings, vector<JoinCondition> &conditions) {
@@ -47,7 +67,7 @@ static bool CreateJoinCondition(Expression &expr, const unordered_set<idx_t> &le
 }
 
 void LogicalComparisonJoin::ExtractJoinConditions(
-    ClientContext &context, JoinType type, unique_ptr<LogicalOperator> &left_child,
+    ClientContext &context, JoinType type, JoinRefType ref_type, unique_ptr<LogicalOperator> &left_child,
     unique_ptr<LogicalOperator> &right_child, const unordered_set<idx_t> &left_bindings,
     const unordered_set<idx_t> &right_bindings, vector<unique_ptr<Expression>> &expressions,
     vector<JoinCondition> &conditions, vector<unique_ptr<Expression>> &arbitrary_expressions) {
@@ -90,7 +110,8 @@ void LogicalComparisonJoin::ExtractJoinConditions(
 
 		{
 			// comparison, check if we can create a comparison JoinCondition
-			if (CreateJoinCondition(*expr, left_bindings, right_bindings, conditions)) {
+			if (IsJoinTypeCondition(ref_type, expr->type) &&
+			    CreateJoinCondition(*expr, left_bindings, right_bindings, conditions)) {
 				// successfully created the join condition
 				continue;
 			}
@@ -99,7 +120,7 @@ void LogicalComparisonJoin::ExtractJoinConditions(
 	}
 }
 
-void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type,
+void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type, JoinRefType ref_type,
                                                   unique_ptr<LogicalOperator> &left_child,
                                                   unique_ptr<LogicalOperator> &right_child,
                                                   vector<unique_ptr<Expression>> &expressions,
@@ -108,11 +129,11 @@ void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinTy
 	unordered_set<idx_t> left_bindings, right_bindings;
 	LogicalJoin::GetTableReferences(*left_child, left_bindings);
 	LogicalJoin::GetTableReferences(*right_child, right_bindings);
-	return ExtractJoinConditions(context, type, left_child, right_child, left_bindings, right_bindings, expressions,
-	                             conditions, arbitrary_expressions);
+	return ExtractJoinConditions(context, type, ref_type, left_child, right_child, left_bindings, right_bindings,
+	                             expressions, conditions, arbitrary_expressions);
 }
 
-void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type,
+void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type, JoinRefType ref_type,
                                                   unique_ptr<LogicalOperator> &left_child,
                                                   unique_ptr<LogicalOperator> &right_child,
                                                   unique_ptr<Expression> condition, vector<JoinCondition> &conditions,
@@ -121,7 +142,7 @@ void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinTy
 	vector<unique_ptr<Expression>> expressions;
 	expressions.push_back(std::move(condition));
 	LogicalFilter::SplitPredicates(expressions);
-	return ExtractJoinConditions(context, type, left_child, right_child, expressions, conditions,
+	return ExtractJoinConditions(context, type, ref_type, left_child, right_child, expressions, conditions,
 	                             arbitrary_expressions);
 }
 
@@ -135,9 +156,6 @@ unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(ClientContext &con
 	bool need_to_consider_arbitrary_expressions = true;
 	switch (reftype) {
 	case JoinRefType::ASOF: {
-		if (!arbitrary_expressions.empty()) {
-			throw BinderException("Invalid ASOF JOIN condition");
-		}
 		need_to_consider_arbitrary_expressions = false;
 		auto asof_idx = conditions.size();
 		for (size_t c = 0; c < conditions.size(); ++c) {
@@ -249,7 +267,7 @@ unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(ClientContext &con
                                                               unique_ptr<Expression> condition) {
 	vector<JoinCondition> conditions;
 	vector<unique_ptr<Expression>> arbitrary_expressions;
-	LogicalComparisonJoin::ExtractJoinConditions(context, type, left_child, right_child, std::move(condition),
+	LogicalComparisonJoin::ExtractJoinConditions(context, type, reftype, left_child, right_child, std::move(condition),
 	                                             conditions, arbitrary_expressions);
 	return LogicalComparisonJoin::CreateJoin(context, type, reftype, std::move(left_child), std::move(right_child),
 	                                         std::move(conditions), std::move(arbitrary_expressions));

package/src/duckdb/src/planner/operator/logical_copy_to_file.cpp

@@ -13,7 +13,7 @@ void LogicalCopyToFile::Serialize(Serializer &serializer) const {
 	serializer.WriteProperty(200, "file_path", file_path);
 	serializer.WriteProperty(201, "use_tmp_file", use_tmp_file);
 	serializer.WriteProperty(202, "filename_pattern", filename_pattern);
-	serializer.WriteProperty(203, "overwrite_or_ignore", overwrite_or_ignore);
+	serializer.WriteProperty(203, "overwrite_or_ignore", overwrite_mode);
 	serializer.WriteProperty(204, "per_thread_output", per_thread_output);
 	serializer.WriteProperty(205, "partition_output", partition_output);
 	serializer.WriteProperty(206, "partition_columns", partition_columns);
@@ -39,7 +39,7 @@ unique_ptr<LogicalOperator> LogicalCopyToFile::Deserialize(Deserializer &deseria
 	auto file_path = deserializer.ReadProperty<string>(200, "file_path");
 	auto use_tmp_file = deserializer.ReadProperty<bool>(201, "use_tmp_file");
 	auto filename_pattern = deserializer.ReadProperty<FilenamePattern>(202, "filename_pattern");
-	auto overwrite_or_ignore = deserializer.ReadProperty<bool>(203, "overwrite_or_ignore");
+	auto overwrite_mode = deserializer.ReadProperty<CopyOverwriteMode>(203, "overwrite_mode");
 	auto per_thread_output = deserializer.ReadProperty<bool>(204, "per_thread_output");
 	auto partition_output = deserializer.ReadProperty<bool>(205, "partition_output");
 	auto partition_columns = deserializer.ReadProperty<vector<idx_t>>(206, "partition_columns");
@@ -86,7 +86,7 @@ unique_ptr<LogicalOperator> LogicalCopyToFile::Deserialize(Deserializer &deseria
 	result->use_tmp_file = use_tmp_file;
 	result->filename_pattern = filename_pattern;
 	result->file_extension = file_extension;
-	result->overwrite_or_ignore = overwrite_or_ignore;
+	result->overwrite_mode = overwrite_mode;
 	result->per_thread_output = per_thread_output;
 	result->partition_output = partition_output;
 	result->partition_columns = partition_columns;

package/src/duckdb/src/storage/storage_manager.cpp

@@ -1,18 +1,20 @@
 #include "duckdb/storage/storage_manager.hpp"
-#include "duckdb/storage/checkpoint_manager.hpp"
-#include "duckdb/storage/in_memory_block_manager.hpp"
-#include "duckdb/storage/single_file_block_manager.hpp"
-#include "duckdb/storage/object_cache.hpp"
 
 #include "duckdb/catalog/catalog.hpp"
 #include "duckdb/common/file_system.hpp"
-#include "duckdb/main/database.hpp"
-#include "duckdb/main/client_context.hpp"
-#include "duckdb/function/function.hpp"
-#include "duckdb/transaction/transaction_manager.hpp"
 #include "duckdb/common/serializer/buffered_file_reader.hpp"
+#include "duckdb/function/function.hpp"
 #include "duckdb/main/attached_database.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/main/database.hpp"
 #include "duckdb/main/database_manager.hpp"
+#include "duckdb/storage/checkpoint_manager.hpp"
+#include "duckdb/storage/in_memory_block_manager.hpp"
+#include "duckdb/storage/object_cache.hpp"
+#include "duckdb/storage/single_file_block_manager.hpp"
+#include "duckdb/transaction/transaction_manager.hpp"
+
+#include "duckdb/storage/storage_extension.hpp"
 
 namespace duckdb {
 
@@ -283,6 +285,9 @@ void SingleFileStorageManager::CreateCheckpoint(CheckpointOptions options) {
 	if (InMemory() || read_only || !load_complete) {
 		return;
 	}
+	if (db.GetStorageExtension()) {
+		db.GetStorageExtension()->OnCheckpointStart(db, options);
+	}
 	auto &config = DBConfig::Get(db);
 	if (GetWALSize() > 0 || config.options.force_checkpoint || options.action == CheckpointAction::FORCE_CHECKPOINT) {
 		// we only need to checkpoint if there is anything in the WAL
@@ -297,6 +302,10 @@ void SingleFileStorageManager::CreateCheckpoint(CheckpointOptions options) {
 	if (options.wal_action == CheckpointWALAction::DELETE_WAL) {
 		ResetWAL();
 	}
+
+	if (db.GetStorageExtension()) {
+		db.GetStorageExtension()->OnCheckpointEnd(db, options);
+	}
 }
 
 DatabaseSize SingleFileStorageManager::GetDatabaseSize() {

package/test/replacement_scan.test.ts

@@ -31,6 +31,7 @@ const invalidResultKeys = (table: string) => {
   } as unknown as sqlite3.ReplacementScanResult;
 };
 
+if (process?.platform != 'win32') {
 describe("replacement scan", () => {
   var db: sqlite3.Database;
   describe("without replacement scan", () => {
@@ -142,3 +143,4 @@ describe("replacement scan", () => {
     });
   });
 });
+}