duckdb 0.7.2-dev832.0 → 0.7.2-dev886.0

package/src/duckdb/src/include/duckdb/common/types/value.hpp

@@ -531,37 +531,6 @@ DUCKDB_API timestamp_t Value::GetValueUnsafe() const;
 template <>
 DUCKDB_API interval_t Value::GetValueUnsafe() const;
 
-template <>
-DUCKDB_API int8_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API int16_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API int32_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API int64_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API hugeint_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API uint8_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API uint16_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API uint32_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API uint64_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API float &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API double &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API date_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API dtime_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API timestamp_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API interval_t &Value::GetReferenceUnsafe();
-
 template <>
 DUCKDB_API bool Value::IsNan(float input);
 template <>

package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp

@@ -67,6 +67,8 @@ private:
 	unique_ptr<LogicalOperator> PushdownSingleJoin(unique_ptr<LogicalOperator> op, unordered_set<idx_t> &left_bindings,
 	                                               unordered_set<idx_t> &right_bindings);
 
+	//! Push any remaining filters into a LogicalFilter at this level
+	unique_ptr<LogicalOperator> PushFinalFilters(unique_ptr<LogicalOperator> op);
 	// Finish pushing down at this operator, creating a LogicalFilter to store any of the stored filters and recursively
 	// pushing down into its children (if any)
 	unique_ptr<LogicalOperator> FinishPushdown(unique_ptr<LogicalOperator> op);

package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp

@@ -8,9 +8,7 @@
 
 #pragma once
 
-#include "duckdb/common/common.hpp"
-#include "duckdb/common/exception.hpp"
-#include "duckdb/common/types/hugeint.hpp"
+#include "duckdb/storage/statistics/numeric_stats_union.hpp"
 #include "duckdb/common/enums/filter_propagate_result.hpp"
 #include "duckdb/common/enums/expression_type.hpp"
 #include "duckdb/common/operator/comparison_operators.hpp"
@@ -23,28 +21,6 @@ class FieldReader;
 struct SelectionVector;
 class Vector;
 
-struct NumericValueUnion {
-	union Val {
-		int8_t boolean;
-		int8_t tinyint;
-		int16_t smallint;
-		int32_t integer;
-		int64_t bigint;
-		uint8_t utinyint;
-		uint16_t usmallint;
-		uint32_t uinteger;
-		uint64_t ubigint;
-		hugeint_t hugeint;
-		float float_;
-		double double_;
-	} value_;
-
-	template <class T>
-	T &GetReferenceUnsafe() {
-		throw InternalException("NumericValueUnion::GetReferenceUnsafe called on unsupported type");
-	}
-};
-
 struct NumericStatsData {
 	//! Whether or not the value has a max value
 	bool has_min;
@@ -109,13 +85,17 @@ struct NumericStats {
 	static void Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count);
 
 	template <class T>
-	static T GetMinUnsafe(const BaseStatistics &stats) {
-		return NumericStats::Min(stats).template GetValueUnsafe<T>();
+	static T GetMin(const BaseStatistics &stats) {
+		return NumericStats::Min(stats).GetValueUnsafe<T>();
 	}
 	template <class T>
-	static T GetMaxUnsafe(const BaseStatistics &stats) {
-		return NumericStats::Max(stats).template GetValueUnsafe<T>();
+	static T GetMax(const BaseStatistics &stats) {
+		return NumericStats::Max(stats).GetValueUnsafe<T>();
 	}
+	template <class T>
+	static T GetMinUnsafe(const BaseStatistics &stats);
+	template <class T>
+	static T GetMaxUnsafe(const BaseStatistics &stats);
 
 private:
 	static NumericStatsData &GetDataUnsafe(BaseStatistics &stats);
@@ -131,27 +111,4 @@ void NumericStats::Update<interval_t>(BaseStatistics &stats, interval_t new_valu
 template <>
 void NumericStats::Update<list_entry_t>(BaseStatistics &stats, list_entry_t new_value);
 
-template <>
-int8_t &NumericValueUnion::GetReferenceUnsafe();
-template <>
-int16_t &NumericValueUnion::GetReferenceUnsafe();
-template <>
-int32_t &NumericValueUnion::GetReferenceUnsafe();
-template <>
-int64_t &NumericValueUnion::GetReferenceUnsafe();
-template <>
-hugeint_t &NumericValueUnion::GetReferenceUnsafe();
-template <>
-uint8_t &NumericValueUnion::GetReferenceUnsafe();
-template <>
-uint16_t &NumericValueUnion::GetReferenceUnsafe();
-template <>
-uint32_t &NumericValueUnion::GetReferenceUnsafe();
-template <>
-uint64_t &NumericValueUnion::GetReferenceUnsafe();
-template <>
-float &NumericValueUnion::GetReferenceUnsafe();
-template <>
-double &NumericValueUnion::GetReferenceUnsafe();
-
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats_union.hpp

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/statistics/numeric_stats_union.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/exception.hpp"
+#include "duckdb/common/types/hugeint.hpp"
+
+namespace duckdb {
+
+struct NumericValueUnion {
+	union Val {
+		bool boolean;
+		int8_t tinyint;
+		int16_t smallint;
+		int32_t integer;
+		int64_t bigint;
+		uint8_t utinyint;
+		uint16_t usmallint;
+		uint32_t uinteger;
+		uint64_t ubigint;
+		hugeint_t hugeint;
+		float float_;
+		double double_;
+	} value_;
+
+	template <class T>
+	T &GetReferenceUnsafe();
+};
+
+template <>
+DUCKDB_API bool &NumericValueUnion::GetReferenceUnsafe();
+template <>
+DUCKDB_API int8_t &NumericValueUnion::GetReferenceUnsafe();
+template <>
+DUCKDB_API int16_t &NumericValueUnion::GetReferenceUnsafe();
+template <>
+DUCKDB_API int32_t &NumericValueUnion::GetReferenceUnsafe();
+template <>
+DUCKDB_API int64_t &NumericValueUnion::GetReferenceUnsafe();
+template <>
+DUCKDB_API hugeint_t &NumericValueUnion::GetReferenceUnsafe();
+template <>
+DUCKDB_API uint8_t &NumericValueUnion::GetReferenceUnsafe();
+template <>
+DUCKDB_API uint16_t &NumericValueUnion::GetReferenceUnsafe();
+template <>
+DUCKDB_API uint32_t &NumericValueUnion::GetReferenceUnsafe();
+template <>
+DUCKDB_API uint64_t &NumericValueUnion::GetReferenceUnsafe();
+template <>
+DUCKDB_API float &NumericValueUnion::GetReferenceUnsafe();
+template <>
+DUCKDB_API double &NumericValueUnion::GetReferenceUnsafe();
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
+#include "duckdb/storage/table/column_data.hpp"
 #include "duckdb/common/unordered_set.hpp"
 
 namespace duckdb {
@@ -28,7 +29,7 @@ struct ColumnCheckpointState {
 
 	RowGroup &row_group;
 	ColumnData &column_data;
-	SegmentTree new_tree;
+	ColumnSegmentTree new_tree;
 	vector<DataPointer> data_pointers;
 	unique_ptr<BaseStatistics> global_stats;
 

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -15,7 +15,8 @@
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/table/persistent_table_data.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
-#include "duckdb/storage/table/column_checkpoint_state.hpp"
+#include "duckdb/storage/table/segment_tree.hpp"
+#include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/mutex.hpp"
 
 namespace duckdb {
@@ -31,10 +32,12 @@ struct TransactionData;
 struct DataTableInfo;
 
 struct ColumnCheckpointInfo {
-	ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
+	explicit ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
 	CompressionType compression_type;
 };
 
+class ColumnSegmentTree : public SegmentTree<ColumnSegment> {};
+
 class ColumnData {
 	friend class ColumnDataCheckpointer;
 
@@ -148,7 +151,7 @@ protected:
 
 protected:
 	//! The segments holding the data of this column segment
-	SegmentTree data;
+	ColumnSegmentTree data;
 	//! The lock for the updates
 	mutex update_lock;
 	//! The updates for this column segment

package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/storage/table/column_data.hpp"
 #include "duckdb/function/compression_function.hpp"
+#include "duckdb/storage/table/column_checkpoint_state.hpp"
 
 namespace duckdb {
 
@@ -25,7 +26,7 @@ public:
 	RowGroup &GetRowGroup();
 	ColumnCheckpointState &GetCheckpointState();
 
-	void Checkpoint(vector<SegmentNode> nodes);
+	void Checkpoint(vector<SegmentNode<ColumnSegment>> nodes);
 
 private:
 	void ScanSegments(const std::function<void(Vector &, idx_t)> &callback);
@@ -40,7 +41,7 @@ private:
 	ColumnCheckpointState &state;
 	bool is_validity;
 	Vector intermediate;
-	vector<SegmentNode> nodes;
+	vector<SegmentNode<ColumnSegment>> nodes;
 	vector<CompressionFunction *> compression_functions;
 	ColumnCheckpointInfo &checkpoint_info;
 };

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -9,7 +9,6 @@
 #pragma once
 
 #include "duckdb/storage/block.hpp"
-#include "duckdb/storage/table/segment_tree.hpp"
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
@@ -17,6 +16,7 @@
 #include "duckdb/storage/storage_lock.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/function/compression_function.hpp"
+#include "duckdb/storage/table/segment_base.hpp"
 
 namespace duckdb {
 class ColumnSegment;
@@ -35,10 +35,12 @@ struct ColumnAppendState;
 enum class ColumnSegmentType : uint8_t { TRANSIENT, PERSISTENT };
 //! TableFilter represents a filter pushed down into the table scan.
 
-class ColumnSegment : public SegmentBase {
+class ColumnSegment : public SegmentBase<ColumnSegment> {
 public:
-	~ColumnSegment() override;
+	~ColumnSegment();
 
+	//! The index within the segment tree
+	idx_t index;
 	//! The database instance
 	DatabaseInstance &db;
 	//! The type stored in the column

package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp

@@ -22,8 +22,11 @@ public:
 	explicit PersistentTableData(idx_t column_count);
 	~PersistentTableData();
 
-	vector<RowGroupPointer> row_groups;
 	TableStatistics table_stats;
+	idx_t total_rows;
+	idx_t row_group_count;
+	block_id_t block_id;
+	idx_t offset;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -9,7 +9,6 @@
 #pragma once
 
 #include "duckdb/common/vector_size.hpp"
-#include "duckdb/storage/table/segment_base.hpp"
 #include "duckdb/storage/table/chunk_info.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
@@ -17,6 +16,7 @@
 #include "duckdb/common/enums/scan_options.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/parser/column_list.hpp"
+#include "duckdb/storage/table/segment_base.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -42,7 +42,7 @@ struct RowGroupWriteData {
 	vector<BaseStatistics> statistics;
 };
 
-class RowGroup : public SegmentBase {
+class RowGroup : public SegmentBase<RowGroup> {
 public:
 	friend class ColumnData;
 	friend class VersionDeleteState;
@@ -58,6 +58,9 @@ public:
 	RowGroup(RowGroup &row_group, idx_t start);
 	~RowGroup();
 
+	//! The index within the segment tree
+	idx_t index;
+
 private:
 	//! The database instance
 	AttachedDatabase &db;
@@ -135,7 +138,7 @@ public:
 	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
 	RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
-	static RowGroupPointer Deserialize(Deserializer &source, const ColumnList &columns);
+	static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
 
 	void InitializeAppend(RowGroupAppendState &append_state);
 	void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -23,6 +23,8 @@ class TableStatistics;
 
 class BoundConstraint;
 
+class RowGroupSegmentTree;
+
 class RowGroupCollection {
 public:
 	RowGroupCollection(shared_ptr<DataTableInfo> info, BlockManager &block_manager, vector<LogicalType> types,
@@ -46,8 +48,8 @@ public:
 	void InitializeCreateIndexScan(CreateIndexScanState &state);
 	void InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids, idx_t start_row,
 	                              idx_t end_row);
-	static bool InitializeScanInRowGroup(CollectionScanState &state, RowGroup *row_group, idx_t vector_index,
-	                                     idx_t max_row);
+	static bool InitializeScanInRowGroup(CollectionScanState &state, ParallelCollectionScanState &parallel_state,
+	                                     idx_t vector_index, idx_t max_row);
 	void InitializeParallelScan(ParallelCollectionScanState &state);
 	bool NextParallelScan(ClientContext &context, ParallelCollectionScanState &state, CollectionScanState &scan_state);
 
@@ -109,7 +111,7 @@ private:
 	vector<LogicalType> types;
 	idx_t row_start;
 	//! The segment trees holding the various row_groups of the table
-	shared_ptr<SegmentTree> row_groups;
+	shared_ptr<RowGroupSegmentTree> row_groups;
 	//! Table statistics
 	TableStatistics stats;
 };

package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp

@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/row_group_segment_tree.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/segment_tree.hpp"
+#include "duckdb/storage/table/row_group.hpp"
+
+namespace duckdb {
+struct DataTableInfo;
+class PersistentTableData;
+class MetaBlockReader;
+
+class RowGroupSegmentTree : public SegmentTree<RowGroup, true> {
+public:
+	RowGroupSegmentTree(DataTableInfo &table_info_p, BlockManager &block_manager_p, vector<LogicalType> column_types_p);
+	~RowGroupSegmentTree() override;
+
+	void Initialize(PersistentTableData &data);
+
+protected:
+	unique_ptr<RowGroup> LoadSegment() override;
+
+	DataTableInfo &info;
+	BlockManager &block_manager;
+	vector<LogicalType> column_types;
+	idx_t current_row_group;
+	idx_t max_row_group;
+	unique_ptr<MetaBlockReader> reader;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp

@@ -25,10 +25,12 @@ class RowGroupCollection;
 class UpdateSegment;
 class TableScanState;
 class ColumnSegment;
+class ColumnSegmentTree;
 class ValiditySegment;
 class TableFilterSet;
 class ColumnData;
 class DuckTransaction;
+class RowGroupSegmentTree;
 
 struct SegmentScanState {
 	virtual ~SegmentScanState() {
@@ -45,6 +47,8 @@ typedef unordered_map<block_id_t, BufferHandle> buffer_handle_set_t;
 struct ColumnScanState {
 	//! The column segment that is currently being scanned
 	ColumnSegment *current = nullptr;
+	//! Column segment tree
+	ColumnSegmentTree *segment_tree = nullptr;
 	//! The current row index of the scan
 	idx_t row_index = 0;
 	//! The internal row index (i.e. the position of the SegmentScanState)
@@ -114,10 +118,12 @@ private:
 class CollectionScanState {
 public:
 	CollectionScanState(TableScanState &parent_p)
-	    : row_group_state(*this), max_row(0), batch_index(0), parent(parent_p) {};
+	    : row_group_state(*this), row_groups(nullptr), max_row(0), batch_index(0), parent(parent_p) {};
 
 	//! The row_group scan state
 	RowGroupScanState row_group_state;
+	//! Row group segment tree
+	RowGroupSegmentTree *row_groups;
 	//! The total maximum row index
 	idx_t max_row;
 	//! The current batch index
@@ -129,6 +135,7 @@ public:
 	AdaptiveFilter *GetAdaptiveFilter();
 	bool Scan(DuckTransaction &transaction, DataChunk &result);
 	bool ScanCommitted(DataChunk &result, TableScanType type);
+	bool ScanCommitted(DataChunk &result, SegmentLock &l, TableScanType type);
 
 private:
 	TableScanState &parent;

package/src/duckdb/src/include/duckdb/storage/table/segment_base.hpp

@@ -13,13 +13,14 @@
 
 namespace duckdb {
 
+template <class T>
 class SegmentBase {
 public:
 	SegmentBase(idx_t start, idx_t count) : start(start), count(count), next(nullptr) {
 	}
 	virtual ~SegmentBase() {
 	}
-	SegmentBase *Next() {
+	T *Next() {
 #ifndef DUCKDB_R_BUILD
 		return next.load();
 #else
@@ -34,9 +35,9 @@ public:
 	//! The next segment after this one
 
 #ifndef DUCKDB_R_BUILD
-	atomic<SegmentBase *> next;
+	atomic<T *> next;
 #else
-	SegmentBase *next;
+	T *next;
 #endif
 };