duckdb 0.10.2-dev0.0 → 0.10.2-dev5.0

package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp

@@ -3,44 +3,139 @@
 #include "duckdb/function/table/read_csv.hpp"
 #include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+#include "duckdb/parser/parsed_data/create_type_info.hpp"
 
 namespace duckdb {
 
-TableCatalogEntry &CSVRejectsTable::GetTable(ClientContext &context) {
+TableCatalogEntry &CSVRejectsTable::GetErrorsTable(ClientContext &context) {
 	auto &temp_catalog = Catalog::GetCatalog(context, TEMP_CATALOG);
-	auto &table_entry = temp_catalog.GetEntry<TableCatalogEntry>(context, TEMP_CATALOG, DEFAULT_SCHEMA, name);
+	auto &table_entry = temp_catalog.GetEntry<TableCatalogEntry>(context, TEMP_CATALOG, DEFAULT_SCHEMA, errors_table);
 	return table_entry;
 }
 
-shared_ptr<CSVRejectsTable> CSVRejectsTable::GetOrCreate(ClientContext &context, const string &name) {
-	auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(name);
+TableCatalogEntry &CSVRejectsTable::GetScansTable(ClientContext &context) {
+	auto &temp_catalog = Catalog::GetCatalog(context, TEMP_CATALOG);
+	auto &table_entry = temp_catalog.GetEntry<TableCatalogEntry>(context, TEMP_CATALOG, DEFAULT_SCHEMA, scan_table);
+	return table_entry;
+}
+
+idx_t CSVRejectsTable::GetCurrentFileIndex(idx_t query_id) {
+	if (current_query_id != query_id) {
+		current_query_id = query_id;
+		current_file_idx = 0;
+	}
+	return current_file_idx++;
+}
+
+shared_ptr<CSVRejectsTable> CSVRejectsTable::GetOrCreate(ClientContext &context, const string &rejects_scan,
+                                                         const string &rejects_error) {
+	// Check that these names can't be the same
+	if (rejects_scan == rejects_error) {
+		throw BinderException("The names of the rejects scan and rejects error tables can't be the same. Use different "
+		                      "names for these tables.");
+	}
+	auto key =
+	    "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(rejects_scan) + "_" + StringUtil::Upper(rejects_error);
 	auto &cache = ObjectCache::GetObjectCache(context);
-	return cache.GetOrCreate<CSVRejectsTable>(key, name);
+	auto &catalog = Catalog::GetCatalog(context, TEMP_CATALOG);
+	auto rejects_scan_exist = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, rejects_scan,
+	                                           OnEntryNotFound::RETURN_NULL) != nullptr;
+	auto rejects_error_exist = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, rejects_error,
+	                                            OnEntryNotFound::RETURN_NULL) != nullptr;
+	if ((rejects_scan_exist || rejects_error_exist) && !cache.Get<CSVRejectsTable>(key)) {
+		std::ostringstream error;
+		if (rejects_scan_exist) {
+			error << "Reject Scan Table name \"" << rejects_scan << "\" is already in use. ";
+		}
+		if (rejects_error_exist) {
+			error << "Reject Error Table name \"" << rejects_error << "\" is already in use. ";
+		}
+		error << "Either drop the used name(s), or give other name options in the CSV Reader function.\n";
+		throw BinderException(error.str());
+	}
+
+	return cache.GetOrCreate<CSVRejectsTable>(key, rejects_scan, rejects_error);
 }
 
 void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData &data) {
 	// (Re)Create the temporary rejects table
 	auto &catalog = Catalog::GetCatalog(context, TEMP_CATALOG);
-	auto info = make_uniq<CreateTableInfo>(TEMP_CATALOG, DEFAULT_SCHEMA, name);
-	info->temporary = true;
-	info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT;
-	info->columns.AddColumn(ColumnDefinition("file", LogicalType::VARCHAR));
-	info->columns.AddColumn(ColumnDefinition("line", LogicalType::BIGINT));
-	info->columns.AddColumn(ColumnDefinition("column", LogicalType::BIGINT));
-	info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR));
-	info->columns.AddColumn(ColumnDefinition("parsed_value", LogicalType::VARCHAR));
-
-	if (!data.options.rejects_recovery_columns.empty()) {
-		child_list_t<LogicalType> recovery_key_components;
-		for (auto &col_name : data.options.rejects_recovery_columns) {
-			recovery_key_components.emplace_back(col_name, LogicalType::VARCHAR);
-		}
-		info->columns.AddColumn(ColumnDefinition("recovery_columns", LogicalType::STRUCT(recovery_key_components)));
-	}
 
-	info->columns.AddColumn(ColumnDefinition("error", LogicalType::VARCHAR));
+	// Create CSV_ERROR_TYPE ENUM
+	string enum_name = "CSV_ERROR_TYPE";
+	Vector order_errors(LogicalType::VARCHAR, 6);
+	order_errors.SetValue(0, "CAST");
+	order_errors.SetValue(1, "MISSING COLUMNS");
+	order_errors.SetValue(2, "TOO MANY COLUMNS");
+	order_errors.SetValue(3, "UNQUOTED VALUE");
+	order_errors.SetValue(4, "LINE SIZE OVER MAXIMUM");
+	order_errors.SetValue(5, "INVALID UNICODE");
+	LogicalType enum_type = LogicalType::ENUM(enum_name, order_errors, 6);
+	auto type_info = make_uniq<CreateTypeInfo>(enum_name, enum_type);
+	type_info->temporary = true;
+	type_info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT;
+	catalog.CreateType(context, *type_info);
 
-	catalog.CreateTable(context, std::move(info));
+	// Create Rejects Scans Table
+	{
+		auto info = make_uniq<CreateTableInfo>(TEMP_CATALOG, DEFAULT_SCHEMA, scan_table);
+		info->temporary = true;
+		info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT;
+		// 0. Scan ID
+		info->columns.AddColumn(ColumnDefinition("scan_id", LogicalType::UBIGINT));
+		// 1. File ID (within the scan)
+		info->columns.AddColumn(ColumnDefinition("file_id", LogicalType::UBIGINT));
+		// 2. File Path
+		info->columns.AddColumn(ColumnDefinition("file_path", LogicalType::VARCHAR));
+		// 3. Delimiter
+		info->columns.AddColumn(ColumnDefinition("delimiter", LogicalType::VARCHAR));
+		// 4. Quote
+		info->columns.AddColumn(ColumnDefinition("quote", LogicalType::VARCHAR));
+		// 5. Escape
+		info->columns.AddColumn(ColumnDefinition("escape", LogicalType::VARCHAR));
+		// 6. NewLine Delimiter
+		info->columns.AddColumn(ColumnDefinition("newline_delimiter", LogicalType::VARCHAR));
+		// 7. Skip Rows
+		info->columns.AddColumn(ColumnDefinition("skip_rows", LogicalType::UINTEGER));
+		// 8. Has Header
+		info->columns.AddColumn(ColumnDefinition("has_header", LogicalType::BOOLEAN));
+		// 9. List<Struct<Column-Name:Types>>
+		info->columns.AddColumn(ColumnDefinition("columns", LogicalType::VARCHAR));
+		// 10. Date Format
+		info->columns.AddColumn(ColumnDefinition("date_format", LogicalType::VARCHAR));
+		// 11. Timestamp Format
+		info->columns.AddColumn(ColumnDefinition("timestamp_format", LogicalType::VARCHAR));
+		// 12. CSV read function with all the options used
+		info->columns.AddColumn(ColumnDefinition("user_arguments", LogicalType::VARCHAR));
+		catalog.CreateTable(context, std::move(info));
+	}
+	{
+		// Create Rejects Error Table
+		auto info = make_uniq<CreateTableInfo>(TEMP_CATALOG, DEFAULT_SCHEMA, errors_table);
+		info->temporary = true;
+		info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT;
+		// 0. Scan ID
+		info->columns.AddColumn(ColumnDefinition("scan_id", LogicalType::UBIGINT));
+		// 1. File ID (within the scan)
+		info->columns.AddColumn(ColumnDefinition("file_id", LogicalType::UBIGINT));
+		// 2. Row Line
+		info->columns.AddColumn(ColumnDefinition("line", LogicalType::UBIGINT));
+		// 3. Byte Position of the start of the line
+		info->columns.AddColumn(ColumnDefinition("line_byte_position", LogicalType::UBIGINT));
+		// 4. Byte Position where error occurred
+		info->columns.AddColumn(ColumnDefinition("byte_position", LogicalType::UBIGINT));
+		// 5. Column Index (If Applicable)
+		info->columns.AddColumn(ColumnDefinition("column_idx", LogicalType::UBIGINT));
+		// 6. Column Name (If Applicable)
+		info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR));
+		// 7. Error Type
+		info->columns.AddColumn(ColumnDefinition("error_type", enum_type));
+		// 8. Original CSV Line
+		info->columns.AddColumn(ColumnDefinition("csv_line", LogicalType::VARCHAR));
+		// 9. Full Error Message
+		info->columns.AddColumn(ColumnDefinition("error_message", LogicalType::VARCHAR));
+		catalog.CreateTable(context, std::move(info));
+	}
 
 	count = 0;
 }

package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp

@@ -62,7 +62,7 @@ struct FixedPreparedBatchData {
 class FixedBatchCopyGlobalState : public GlobalSinkState {
 public:
 	// heuristic - we need at least 4MB of cache space per column per thread we launch
-	static constexpr const idx_t MINIMUM_MEMORY_PER_COLUMN_PER_THREAD = 4 * 1024 * 1024;
+	static constexpr const idx_t MINIMUM_MEMORY_PER_COLUMN_PER_THREAD = 4ULL * 1024ULL * 1024ULL;
 
 public:
 	explicit FixedBatchCopyGlobalState(ClientContext &context_p, unique_ptr<GlobalFunctionData> global_state,
@@ -117,7 +117,7 @@ public:
 	}
 };
 
-enum class FixedBatchCopyState { SINKING_DATA = 1, PROCESSING_TASKS = 2 };
+enum class FixedBatchCopyState : uint8_t { SINKING_DATA = 1, PROCESSING_TASKS = 2 };
 
 class FixedBatchCopyLocalState : public LocalSinkState {
 public:

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -125,8 +125,9 @@ public:
 
 class BatchInsertGlobalState : public GlobalSinkState {
 public:
-	explicit BatchInsertGlobalState(ClientContext &context, DuckTableEntry &table, idx_t initial_memory)
-	    : memory_manager(context, initial_memory), table(table), insert_count(0), optimistically_written(false) {
+	explicit BatchInsertGlobalState(ClientContext &context, DuckTableEntry &table, idx_t minimum_memory_per_thread)
+	    : memory_manager(context, minimum_memory_per_thread), table(table), insert_count(0),
+	      optimistically_written(false), minimum_memory_per_thread(minimum_memory_per_thread) {
 	}
 
 	BatchMemoryManager memory_manager;
@@ -137,6 +138,7 @@ public:
 	vector<RowGroupBatchEntry> collections;
 	idx_t next_start = 0;
 	atomic<bool> optimistically_written;
+	idx_t minimum_memory_per_thread;
 
 	static bool ReadyToMerge(idx_t count);
 	void ScheduleMergeTasks(idx_t min_batch_index);
@@ -146,6 +148,13 @@ public:
 	void AddCollection(ClientContext &context, idx_t batch_index, idx_t min_batch_index,
 	                   unique_ptr<RowGroupCollection> current_collection,
 	                   optional_ptr<OptimisticDataWriter> writer = nullptr);
+
+	idx_t MaxThreads(idx_t source_max_threads) override {
+		// try to request 4MB per column per thread
+		memory_manager.SetMemorySize(source_max_threads * minimum_memory_per_thread);
+		// cap the concurrent threads working on this task based on the amount of available memory
+		return MinValue<idx_t>(source_max_threads, memory_manager.AvailableMemory() / minimum_memory_per_thread + 1);
+	}
 };
 
 class BatchInsertLocalState : public LocalSinkState {
@@ -376,9 +385,9 @@ unique_ptr<GlobalSinkState> PhysicalBatchInsert::GetGlobalSinkState(ClientContex
 		table = insert_table.get_mutable();
 	}
 	// heuristic - we start off by allocating 4MB of cache space per column
-	static constexpr const idx_t MINIMUM_MEMORY_PER_COLUMN = 4 * 1024 * 1024;
-	auto initial_memory = table->GetColumns().PhysicalColumnCount() * MINIMUM_MEMORY_PER_COLUMN;
-	auto result = make_uniq<BatchInsertGlobalState>(context, table->Cast<DuckTableEntry>(), initial_memory);
+	static constexpr const idx_t MINIMUM_MEMORY_PER_COLUMN = 4ULL * 1024ULL * 1024ULL;
+	auto minimum_memory_per_thread = table->GetColumns().PhysicalColumnCount() * MINIMUM_MEMORY_PER_COLUMN;
+	auto result = make_uniq<BatchInsertGlobalState>(context, table->Cast<DuckTableEntry>(), minimum_memory_per_thread);
 	return std::move(result);
 }
 

package/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp

@@ -25,7 +25,7 @@ PhysicalCopyDatabase::~PhysicalCopyDatabase() {
 //===--------------------------------------------------------------------===//
 SourceResultType PhysicalCopyDatabase::GetData(ExecutionContext &context, DataChunk &chunk,
                                                OperatorSourceInput &input) const {
-	auto &catalog = info->to_database;
+	auto &catalog = Catalog::GetCatalog(context.client, info->target_database);
 	for (auto &create_info : info->entries) {
 		switch (create_info->type) {
 		case CatalogType::SCHEMA_ENTRY:

package/src/duckdb/src/execution/operator/persistent/physical_export.cpp

@@ -22,15 +22,15 @@ static void WriteCatalogEntries(stringstream &ss, vector<reference<CatalogEntry>
 		if (entry.get().internal) {
 			continue;
 		}
-		ss << entry.get().ToSQL() << std::endl;
+		ss << entry.get().ToSQL() << '\n';
 	}
-	ss << std::endl;
+	ss << '\n';
 }
 
 static void WriteStringStreamToFile(FileSystem &fs, stringstream &ss, const string &path) {
 	auto ss_string = ss.str();
-	auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW,
-	                          FileLockType::WRITE_LOCK);
+	auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW |
+	                                    FileLockType::WRITE_LOCK);
 	fs.Write(*handle, (void *)ss_string.c_str(), ss_string.size());
 	handle.reset();
 }
@@ -77,7 +77,7 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info,
 			throw NotImplementedException("FIXME: serialize list of options");
 		}
 	}
-	ss << ");" << std::endl;
+	ss << ");" << '\n';
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/execution/operator/schema/physical_attach.cpp

@@ -99,7 +99,7 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c
 	// get the database type and attach the database
 	db_manager.GetDatabaseType(context.client, db_type, *info, config, unrecognized_option);
 	auto attached_db = db_manager.AttachDatabase(context.client, *info, db_type, access_mode);
-	attached_db->Initialize();
+	attached_db->Initialize(&context.client);
 	return SourceResultType::FINISHED;
 }
 

package/src/duckdb/src/execution/physical_plan/plan_simple.cpp

@@ -25,15 +25,6 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalSimple &op
 	case LogicalOperatorType::LOGICAL_TRANSACTION:
 		return make_uniq<PhysicalTransaction>(unique_ptr_cast<ParseInfo, TransactionInfo>(std::move(op.info)),
 		                                      op.estimated_cardinality);
-	case LogicalOperatorType::LOGICAL_VACUUM: {
-		auto result = make_uniq<PhysicalVacuum>(unique_ptr_cast<ParseInfo, VacuumInfo>(std::move(op.info)),
-		                                        op.estimated_cardinality);
-		if (!op.children.empty()) {
-			auto child = CreatePlan(*op.children[0]);
-			result->children.push_back(std::move(child));
-		}
-		return std::move(result);
-	}
 	case LogicalOperatorType::LOGICAL_LOAD:
 		return make_uniq<PhysicalLoad>(unique_ptr_cast<ParseInfo, LoadInfo>(std::move(op.info)),
 		                               op.estimated_cardinality);

package/src/duckdb/src/execution/physical_plan/plan_vacuum.cpp

@@ -0,0 +1,18 @@
+#include "duckdb/execution/operator/helper/physical_vacuum.hpp"
+#include "duckdb/execution/physical_plan_generator.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/operator/logical_vacuum.hpp"
+
+namespace duckdb {
+
+unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalVacuum &op) {
+	auto result = make_uniq<PhysicalVacuum>(unique_ptr_cast<ParseInfo, VacuumInfo>(std::move(op.info)), op.table,
+	                                        std::move(op.column_id_map), op.estimated_cardinality);
+	if (!op.children.empty()) {
+		auto child = CreatePlan(*op.children[0]);
+		result->children.push_back(std::move(child));
+	}
+	return std::move(result);
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/physical_plan_generator.cpp

@@ -15,7 +15,7 @@ namespace duckdb {
 
 class DependencyExtractor : public LogicalOperatorVisitor {
 public:
-	explicit DependencyExtractor(DependencyList &dependencies) : dependencies(dependencies) {
+	explicit DependencyExtractor(LogicalDependencyList &dependencies) : dependencies(dependencies) {
 	}
 
 protected:
@@ -28,7 +28,7 @@ protected:
 	}
 
 private:
-	DependencyList &dependencies;
+	LogicalDependencyList &dependencies;
 };
 
 PhysicalPlanGenerator::PhysicalPlanGenerator(ClientContext &context) : context(context) {
@@ -176,10 +176,12 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalOperator &
 	case LogicalOperatorType::LOGICAL_PRAGMA:
 		plan = CreatePlan(op.Cast<LogicalPragma>());
 		break;
+	case LogicalOperatorType::LOGICAL_VACUUM:
+		plan = CreatePlan(op.Cast<LogicalVacuum>());
+		break;
 	case LogicalOperatorType::LOGICAL_TRANSACTION:
 	case LogicalOperatorType::LOGICAL_ALTER:
 	case LogicalOperatorType::LOGICAL_DROP:
-	case LogicalOperatorType::LOGICAL_VACUUM:
 	case LogicalOperatorType::LOGICAL_LOAD:
 	case LogicalOperatorType::LOGICAL_ATTACH:
 	case LogicalOperatorType::LOGICAL_DETACH: