duckdb 0.7.2-dev1034.0 → 0.7.2-dev1146.0

package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp

@@ -72,12 +72,11 @@ struct ColumnScanState {
 	idx_t last_offset = 0;
 
 public:
+	void Initialize(const LogicalType &type);
 	//! Move the scan state forward by "count" rows (including all child states)
 	void Next(idx_t count);
 	//! Move ONLY this state forward by "count" rows (i.e. not the child states)
 	void NextInternal(idx_t count);
-	//! Move the scan state forward by STANDARD_VECTOR_SIZE rows
-	void NextVector();
 };
 
 struct ColumnFetchState {
@@ -89,39 +88,18 @@ struct ColumnFetchState {
 	BufferHandle &GetOrInsertHandle(ColumnSegment &segment);
 };
 
-class RowGroupScanState {
+class CollectionScanState {
 public:
-	RowGroupScanState(CollectionScanState &parent_p)
-	    : row_group(nullptr), vector_index(0), max_row(0), parent(parent_p) {
-	}
+	CollectionScanState(TableScanState &parent_p);
 
 	//! The current row_group we are scanning
-	RowGroup *row_group = nullptr;
+	RowGroup *row_group;
 	//! The vector index within the row_group
-	idx_t vector_index = 0;
-	//! The maximum row index of this row_group scan
-	idx_t max_row = 0;
+	idx_t vector_index;
+	//! The maximum row within the row group
+	idx_t max_row_group_row;
 	//! Child column scans
 	unique_ptr<ColumnScanState[]> column_scans;
-
-public:
-	const vector<column_t> &GetColumnIds();
-	TableFilterSet *GetFilters();
-	AdaptiveFilter *GetAdaptiveFilter();
-	idx_t GetParentMaxRow();
-
-private:
-	//! The parent scan state
-	CollectionScanState &parent;
-};
-
-class CollectionScanState {
-public:
-	CollectionScanState(TableScanState &parent_p)
-	    : row_group_state(*this), row_groups(nullptr), max_row(0), batch_index(0), parent(parent_p) {};
-
-	//! The row_group scan state
-	RowGroupScanState row_group_state;
 	//! Row group segment tree
 	RowGroupSegmentTree *row_groups;
 	//! The total maximum row index
@@ -130,6 +108,7 @@ public:
 	idx_t batch_index;
 
 public:
+	void Initialize(const vector<LogicalType> &types);
 	const vector<column_t> &GetColumnIds();
 	TableFilterSet *GetFilters();
 	AdaptiveFilter *GetAdaptiveFilter();
@@ -167,12 +146,16 @@ private:
 };
 
 struct ParallelCollectionScanState {
+	ParallelCollectionScanState();
+
 	//! The row group collection we are scanning
 	RowGroupCollection *collection;
 	RowGroup *current_row_group;
 	idx_t vector_index;
 	idx_t max_row;
 	idx_t batch_index;
+	atomic<idx_t> processed_rows;
+	mutex lock;
 };
 
 struct ParallelTableScanState {

package/src/duckdb/src/include/duckdb/storage/table/segment_base.hpp

@@ -18,8 +18,6 @@ class SegmentBase {
 public:
 	SegmentBase(idx_t start, idx_t count) : start(start), count(count), next(nullptr) {
 	}
-	virtual ~SegmentBase() {
-	}
 	T *Next() {
 #ifndef DUCKDB_R_BUILD
 		return next.load();
@@ -33,12 +31,13 @@ public:
 	//! The amount of entries in this storage chunk
 	atomic<idx_t> count;
 	//! The next segment after this one
-
 #ifndef DUCKDB_R_BUILD
 	atomic<T *> next;
 #else
 	T *next;
 #endif
+	//! The index within the segment tree
+	idx_t index;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp

@@ -56,7 +56,7 @@ public:
 		if (nodes.empty()) {
 			LoadNextSegment(l);
 		}
-		return nodes.empty() ? nullptr : nodes[0].node.get();
+		return GetRootSegmentInternal();
 	}
 	//! Obtains ownership of the data of the segment tree
 	vector<SegmentNode<T>> MoveSegments(SegmentLock &l) {
@@ -258,6 +258,10 @@ private:
 	mutex node_lock;
 
 private:
+	T *GetRootSegmentInternal() {
+		return nodes.empty() ? nullptr : nodes[0].node.get();
+	}
+
 	class SegmentIterationHelper {
 	public:
 		explicit SegmentIterationHelper(SegmentTree &tree) : tree(tree) {
@@ -304,6 +308,9 @@ private:
 
 	//! Load the next segment, if there are any left to load
 	bool LoadNextSegment(SegmentLock &l) {
+		if (!SUPPORTS_LAZY_LOADING) {
+			return false;
+		}
 		if (finished_loading) {
 			return false;
 		}
@@ -317,6 +324,9 @@ private:
 
 	//! Load all segments, if there are any left to load
 	void LoadAllSegments(SegmentLock &l) {
+		if (!SUPPORTS_LAZY_LOADING) {
+			return;
+		}
 		while (LoadNextSegment(l))
 			;
 	}

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -59,10 +59,6 @@ public:
 	void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result) override;
 
 	void Verify(RowGroup &parent) override;
-
-private:
-	template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
-	void TemplatedScan(Transaction *transaction, ColumnScanState &state, Vector &result);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -9,7 +9,6 @@
 #pragma once
 
 #include "duckdb/storage/table/row_group_collection.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table/table_index_list.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
 
@@ -18,6 +17,7 @@ class AttachedDatabase;
 class DataTable;
 class Transaction;
 class WriteAheadLog;
+struct LocalAppendState;
 struct TableAppendState;
 
 class OptimisticDataWriter {
@@ -118,6 +118,9 @@ public:
 
 public:
 	struct CommitState {
+		CommitState();
+		~CommitState();
+
 		unordered_map<DataTable *, unique_ptr<TableAppendState>> append_states;
 	};
 

package/src/duckdb/src/include/duckdb.h

@@ -1710,6 +1710,16 @@ Adds a parameter to the table function.
 */
 DUCKDB_API void duckdb_table_function_add_parameter(duckdb_table_function table_function, duckdb_logical_type type);
 
+/*!
+Adds a named parameter to the table function.
+
+* table_function: The table function
+* name: The name of the parameter
+* type: The type of the parameter to add.
+*/
+DUCKDB_API void duckdb_table_function_add_named_parameter(duckdb_table_function table_function, const char *name,
+                                                          duckdb_logical_type type);
+
 /*!
 Assigns extra information to the table function that can be fetched during binding, etc.
 
@@ -1818,6 +1828,17 @@ The result must be destroyed with `duckdb_destroy_value`.
 */
 DUCKDB_API duckdb_value duckdb_bind_get_parameter(duckdb_bind_info info, idx_t index);
 
+/*!
+Retrieves a named parameter with the given name.
+
+The result must be destroyed with `duckdb_destroy_value`.
+
+* info: The info object
+* name: The name of the parameter
+* returns: The value of the parameter. Must be destroyed with `duckdb_destroy_value`.
+*/
+DUCKDB_API duckdb_value duckdb_bind_get_named_parameter(duckdb_bind_info info, const char *name);
+
 /*!
 Sets the user-provided bind data in the bind object. This object can be retrieved again during execution.
 

package/src/duckdb/src/main/capi/table_function-c.cpp

@@ -208,6 +208,16 @@ void duckdb_table_function_add_parameter(duckdb_table_function function, duckdb_
 	tf->arguments.push_back(*logical_type);
 }
 
+void duckdb_table_function_add_named_parameter(duckdb_table_function function, const char *name,
+                                               duckdb_logical_type type) {
+	if (!function || !type) {
+		return;
+	}
+	auto tf = (duckdb::TableFunction *)function;
+	auto logical_type = (duckdb::LogicalType *)type;
+	tf->named_parameters.insert({name, *logical_type});
+}
+
 void duckdb_table_function_set_extra_info(duckdb_table_function function, void *extra_info,
                                           duckdb_delete_callback_t destroy) {
 	if (!function) {
@@ -319,6 +329,19 @@ duckdb_value duckdb_bind_get_parameter(duckdb_bind_info info, idx_t index) {
 	return reinterpret_cast<duckdb_value>(new duckdb::Value(bind_info->input.inputs[index]));
 }
 
+duckdb_value duckdb_bind_get_named_parameter(duckdb_bind_info info, const char *name) {
+	if (!info || !name) {
+		return nullptr;
+	}
+	auto bind_info = (duckdb::CTableInternalBindInfo *)info;
+	auto t = bind_info->input.named_parameters.find(name);
+	if (t == bind_info->input.named_parameters.end()) {
+		return nullptr;
+	} else {
+		return reinterpret_cast<duckdb_value>(new duckdb::Value(t->second));
+	}
+}
+
 void duckdb_bind_set_bind_data(duckdb_bind_info info, void *bind_data, duckdb_delete_callback_t destroy) {
 	if (!info) {
 		return;

package/src/duckdb/src/main/settings/settings.cpp

@@ -454,11 +454,15 @@ Value CustomExtensionRepository::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 
 void EnableProgressBarSetting::ResetLocal(ClientContext &context) {
-	ClientConfig::GetConfig(context).enable_progress_bar = ClientConfig().enable_progress_bar;
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.enable_progress_bar = ClientConfig().enable_progress_bar;
 }
 
 void EnableProgressBarSetting::SetLocal(ClientContext &context, const Value &input) {
-	ClientConfig::GetConfig(context).enable_progress_bar = input.GetValue<bool>();
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.enable_progress_bar = input.GetValue<bool>();
 }
 
 Value EnableProgressBarSetting::GetSetting(ClientContext &context) {
@@ -469,11 +473,15 @@ Value EnableProgressBarSetting::GetSetting(ClientContext &context) {
 // Enable Progress Bar Print
 //===--------------------------------------------------------------------===//
 void EnableProgressBarPrintSetting::SetLocal(ClientContext &context, const Value &input) {
-	ClientConfig::GetConfig(context).print_progress_bar = input.GetValue<bool>();
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.print_progress_bar = input.GetValue<bool>();
 }
 
 void EnableProgressBarPrintSetting::ResetLocal(ClientContext &context) {
-	ClientConfig::GetConfig(context).print_progress_bar = ClientConfig().print_progress_bar;
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.print_progress_bar = ClientConfig().print_progress_bar;
 }
 
 Value EnableProgressBarPrintSetting::GetSetting(ClientContext &context) {
@@ -878,13 +886,17 @@ Value ProfilingModeSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 
 void ProgressBarTimeSetting::ResetLocal(ClientContext &context) {
-	ClientConfig::GetConfig(context).wait_time = ClientConfig().wait_time;
-	ClientConfig::GetConfig(context).enable_progress_bar = ClientConfig().enable_progress_bar;
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.wait_time = ClientConfig().wait_time;
+	config.enable_progress_bar = ClientConfig().enable_progress_bar;
 }
 
 void ProgressBarTimeSetting::SetLocal(ClientContext &context, const Value &input) {
-	ClientConfig::GetConfig(context).wait_time = input.GetValue<int32_t>();
-	ClientConfig::GetConfig(context).enable_progress_bar = true;
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.wait_time = input.GetValue<int32_t>();
+	config.enable_progress_bar = true;
 }
 
 Value ProgressBarTimeSetting::GetSetting(ClientContext &context) {

package/src/duckdb/src/optimizer/filter_combiner.cpp

@@ -2,21 +2,18 @@
 
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/planner/expression.hpp"
-#include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/expression/bound_between_expression.hpp"
+#include "duckdb/planner/expression/bound_cast_expression.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
 #include "duckdb/planner/expression/bound_comparison_expression.hpp"
 #include "duckdb/planner/expression/bound_conjunction_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
-#include "duckdb/planner/operator/logical_empty_result.hpp"
-#include "duckdb/planner/operator/logical_filter.hpp"
+#include "duckdb/planner/expression/bound_operator_expression.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/planner/filter/constant_filter.hpp"
 #include "duckdb/planner/filter/null_filter.hpp"
 #include "duckdb/optimizer/optimizer.hpp"
-#include "duckdb/common/operator/cast_operators.hpp"
-#include "duckdb/planner/filter/conjunction_filter.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp

@@ -8,6 +8,8 @@
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 
+#include <cmath>
+
 namespace duckdb {
 
 static TableCatalogEntry *GetCatalogTableEntry(LogicalOperator *op) {

package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp

@@ -7,6 +7,7 @@
 #include "duckdb/planner/operator/list.hpp"
 
 #include <algorithm>
+#include <cmath>
 
 namespace std {
 

package/src/duckdb/src/parallel/meta_pipeline.cpp

@@ -156,9 +156,6 @@ Pipeline *MetaPipeline::CreateUnionPipeline(Pipeline &current, bool order_matter
 void MetaPipeline::CreateChildPipeline(Pipeline &current, PhysicalOperator *op, Pipeline *last_pipeline) {
 	// rule 2: 'current' must be fully built (down to the source) before creating the child pipeline
 	D_ASSERT(current.source);
-	if (HasRecursiveCTE()) {
-		throw NotImplementedException("Child pipelines are not supported in recursive CTEs yet");
-	}
 
 	// create the child pipeline (same batch index)
 	pipelines.emplace_back(state.CreateChildPipeline(executor, current, op));

package/src/duckdb/src/parser/transform/expression/transform_function.cpp

@@ -1,7 +1,9 @@
+#include "duckdb/common/serializer/enum_serializer.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/to_string.hpp"
 #include "duckdb/parser/expression/case_expression.hpp"
 #include "duckdb/parser/expression/cast_expression.hpp"
+#include "duckdb/parser/expression/constant_expression.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
 
 #include "duckdb/parser/expression/operator_expression.hpp"
@@ -310,6 +312,26 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 		coalesce_op->children.push_back(std::move(children[0]));
 		coalesce_op->children.push_back(std::move(children[1]));
 		return std::move(coalesce_op);
+	} else if (lowercase_name == "list" && order_bys->orders.size() == 1) {
+		// list(expr ORDER BY expr <sense> <nulls>) => list_sort(list(expr), <sense>, <nulls>)
+		if (children.size() != 1) {
+			throw ParserException("Wrong number of arguments to LIST.");
+		}
+		auto arg_expr = children[0].get();
+		auto &order_by = order_bys->orders[0];
+		if (arg_expr->Equals(order_by.expression.get())) {
+			auto sense = make_unique<ConstantExpression>(EnumSerializer::EnumToString(order_by.type));
+			auto nulls = make_unique<ConstantExpression>(EnumSerializer::EnumToString(order_by.null_order));
+			order_bys = nullptr;
+			auto unordered = make_unique<FunctionExpression>(
+			    catalog, schema, lowercase_name.c_str(), std::move(children), std::move(filter_expr),
+			    std::move(order_bys), root->agg_distinct, false, root->export_state);
+			lowercase_name = "list_sort";
+			children.clear();
+			children.emplace_back(std::move(unordered));
+			children.emplace_back(std::move(sense));
+			children.emplace_back(std::move(nulls));
+		}
 	}
 
 	auto function = make_unique<FunctionExpression>(std::move(catalog), std::move(schema), lowercase_name.c_str(),

package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/constraints/list.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
+#include "duckdb/planner/expression/bound_function_expression.hpp"
 #include "duckdb/planner/expression_binder/check_binder.hpp"
 #include "duckdb/planner/expression_binder/constant_binder.hpp"
 #include "duckdb/planner/parsed_data/bound_create_table_info.hpp"

package/src/duckdb/src/storage/compression/bitpacking.cpp

@@ -1,7 +1,6 @@
 #include "duckdb/common/bitpacking.hpp"
 
 #include "duckdb/common/limits.hpp"
-#include "duckdb/common/types/null_value.hpp"
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
@@ -11,6 +10,7 @@
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"
 #include "duckdb/storage/compression/bitpacking.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 #include <functional>
 

package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp

@@ -9,6 +9,7 @@
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -64,7 +65,7 @@ void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
 	auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
 	if (type.InternalType() == PhysicalType::VARCHAR) {
 		auto &state = (UncompressedStringSegmentState &)*compressed_segment->GetSegmentState();
-		state.overflow_writer = make_unique<WriteOverflowStringsToDisk>(checkpointer.GetColumnData().block_manager);
+		state.overflow_writer = make_unique<WriteOverflowStringsToDisk>(checkpointer.GetColumnData().GetBlockManager());
 	}
 	current_segment = std::move(compressed_segment);
 	current_segment->InitializeAppend(append_state);

package/src/duckdb/src/storage/compression/numeric_constant.cpp

@@ -1,10 +1,10 @@
 #include "duckdb/function/compression/compression.hpp"
-#include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/common/types/vector.hpp"
 
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/storage/segment/uncompressed.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/storage/compression/rle.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/common/types/null_value.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 #include <functional>
 
 namespace duckdb {

package/src/duckdb/src/storage/compression/validity_uncompressed.cpp

@@ -6,7 +6,7 @@
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
-#include "duckdb/main/config.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/storage/data_table.cpp

@@ -5,7 +5,6 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/common/sort/sort.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/parser/constraints/list.hpp"
@@ -23,6 +22,8 @@
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/common/types/conflict_manager.hpp"
 #include "duckdb/common/types/constraint_conflict_info.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -728,8 +729,7 @@ void DataTable::ScanTableSegment(idx_t row_start, idx_t count, const std::functi
 	CreateIndexScanState state;
 
 	InitializeScanWithOffset(state, column_ids, row_start, row_start + count);
-	auto row_start_aligned = state.table_state.row_group_state.row_group->start +
-	                         state.table_state.row_group_state.vector_index * STANDARD_VECTOR_SIZE;
+	auto row_start_aligned = state.table_state.row_group->start + state.table_state.vector_index * STANDARD_VECTOR_SIZE;
 
 	idx_t current_row = row_start_aligned;
 	while (current_row < end) {

package/src/duckdb/src/storage/local_storage.cpp

@@ -11,6 +11,7 @@
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/storage/table_io_manager.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -355,6 +356,12 @@ LocalStorage::LocalStorage(ClientContext &context, DuckTransaction &transaction)
     : context(context), transaction(transaction) {
 }
 
+LocalStorage::CommitState::CommitState() {
+}
+
+LocalStorage::CommitState::~CommitState() {
+}
+
 LocalStorage &LocalStorage::Get(DuckTransaction &transaction) {
 	return transaction.GetLocalStorage();
 }

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 48;
+const uint64_t VERSION_NUMBER = 49;
 
 struct StorageVersionInfo {
 	const char *version_name;