duckdb 0.7.2-dev12.0 → 0.7.2-dev1244.0

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -1,6 +1,9 @@
 #include "duckdb/storage/table/list_column_data.hpp"
-#include "duckdb/storage/statistics/list_statistics.hpp"
+#include "duckdb/storage/statistics/list_stats.hpp"
 #include "duckdb/transaction/transaction.hpp"
+#include "duckdb/storage/table/column_checkpoint_state.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -29,25 +32,21 @@ void ListColumnData::InitializeScan(ColumnScanState &state) {
 	ColumnData::InitializeScan(state);
 
 	// initialize the validity segment
-	ColumnScanState validity_state;
-	validity.InitializeScan(validity_state);
-	state.child_states.push_back(std::move(validity_state));
+	D_ASSERT(state.child_states.size() == 2);
+	validity.InitializeScan(state.child_states[0]);
 
 	// initialize the child scan
-	ColumnScanState child_state;
-	child_column->InitializeScan(child_state);
-	state.child_states.push_back(std::move(child_state));
+	child_column->InitializeScan(state.child_states[1]);
 }
 
-list_entry_t ListColumnData::FetchListEntry(idx_t row_idx) {
-	auto segment = (ColumnSegment *)data.GetSegment(row_idx);
+uint64_t ListColumnData::FetchListOffset(idx_t row_idx) {
+	auto segment = data.GetSegment(row_idx);
 	ColumnFetchState fetch_state;
 	Vector result(type, 1);
 	segment->FetchRow(fetch_state, row_idx, result, 0);
 
 	// initialize the child scan with the required offset
-	auto list_data = FlatVector::GetData<list_entry_t>(result);
-	return list_data[0];
+	return FlatVector::GetData<uint64_t>(result)[0];
 }
 
 void ListColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) {
@@ -58,20 +57,16 @@ void ListColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_
 	ColumnData::InitializeScanWithOffset(state, row_idx);
 
 	// initialize the validity segment
-	ColumnScanState validity_state;
-	validity.InitializeScanWithOffset(validity_state, row_idx);
-	state.child_states.push_back(std::move(validity_state));
+	D_ASSERT(state.child_states.size() == 2);
+	validity.InitializeScanWithOffset(state.child_states[0], row_idx);
 
 	// we need to read the list at position row_idx to get the correct row offset of the child
-	auto list_entry = FetchListEntry(row_idx);
-	auto child_offset = list_entry.offset;
+	auto child_offset = row_idx == start ? 0 : FetchListOffset(row_idx - 1);
 
 	D_ASSERT(child_offset <= child_column->GetMaxEntry());
-	ColumnScanState child_state;
 	if (child_offset < child_column->GetMaxEntry()) {
-		child_column->InitializeScanWithOffset(child_state, start + child_offset);
+		child_column->InitializeScanWithOffset(state.child_states[1], start + child_offset);
 	}
-	state.child_states.push_back(std::move(child_state));
 }
 
 idx_t ListColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) {
@@ -89,26 +84,26 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	// updates not supported for lists
 	D_ASSERT(!updates);
 
-	idx_t scan_count = ScanVector(state, result, count);
+	Vector offset_vector(LogicalType::UBIGINT, count);
+	idx_t scan_count = ScanVector(state, offset_vector, count);
 	D_ASSERT(scan_count > 0);
 	validity.ScanCount(state.child_states[0], result, count);
 
-	auto data = FlatVector::GetData<list_entry_t>(result);
-	auto first_entry = data[0];
+	auto data = FlatVector::GetData<uint64_t>(offset_vector);
 	auto last_entry = data[scan_count - 1];
 
-#ifdef DEBUG
-	for (idx_t i = 1; i < scan_count; i++) {
-		D_ASSERT(data[i].offset == data[i - 1].offset + data[i - 1].length);
-	}
-#endif
 	// shift all offsets so they are 0 at the first entry
+	auto result_data = FlatVector::GetData<list_entry_t>(result);
+	auto base_offset = state.last_offset;
+	idx_t current_offset = 0;
 	for (idx_t i = 0; i < scan_count; i++) {
-		data[i].offset -= first_entry.offset;
+		result_data[i].offset = current_offset;
+		result_data[i].length = data[i] - current_offset - base_offset;
+		current_offset += result_data[i].length;
 	}
 
-	D_ASSERT(last_entry.offset >= first_entry.offset);
-	idx_t child_scan_count = last_entry.offset + last_entry.length - first_entry.offset;
+	D_ASSERT(last_entry >= base_offset);
+	idx_t child_scan_count = last_entry - base_offset;
 	ListVector::Reserve(result, child_scan_count);
 
 	if (child_scan_count > 0) {
@@ -118,6 +113,7 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 		             child_column->start + child_column->GetMaxEntry());
 		child_column->ScanCount(state.child_states[1], child_entry, child_scan_count);
 	}
+	state.last_offset = last_entry;
 
 	ListVector::SetListSize(result, child_scan_count);
 	return scan_count;
@@ -130,19 +126,19 @@ void ListColumnData::Skip(ColumnScanState &state, idx_t count) {
 	// we need to read the list entries/offsets to figure out how much to skip
 	// note that we only need to read the first and last entry
 	// however, let's just read all "count" entries for now
-	auto data = unique_ptr<list_entry_t[]>(new list_entry_t[count]);
-	Vector result(type, (data_ptr_t)data.get());
+	Vector result(LogicalType::UBIGINT, count);
 	idx_t scan_count = ScanVector(state, result, count);
 	if (scan_count == 0) {
 		return;
 	}
 
-	auto &first_entry = data[0];
-	auto &last_entry = data[scan_count - 1];
-	idx_t child_scan_count = last_entry.offset + last_entry.length - first_entry.offset;
+	auto data = FlatVector::GetData<uint64_t>(result);
+	auto last_entry = data[scan_count - 1];
+	idx_t child_scan_count = last_entry - state.last_offset;
 	if (child_scan_count == 0) {
 		return;
 	}
+	state.last_offset = last_entry;
 
 	// skip the child state forward by the child_scan_count
 	child_column->Skip(state.child_states[1], child_scan_count);
@@ -163,10 +159,8 @@ void ListColumnData::InitializeAppend(ColumnAppendState &state) {
 	state.child_appends.push_back(std::move(child_append_state));
 }
 
-void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, Vector &vector, idx_t count) {
+void ListColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count) {
 	D_ASSERT(count > 0);
-	auto &stats = (ListStatistics &)stats_p;
-
 	UnifiedVectorFormat list_data;
 	vector.ToUnifiedFormat(count, list_data);
 	auto &list_validity = list_data.validity;
@@ -177,8 +171,8 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 	idx_t child_count = 0;
 
 	ValidityMask append_mask(count);
-	auto append_offsets = unique_ptr<list_entry_t[]>(new list_entry_t[count]);
-	bool child_contiguous = false;
+	auto append_offsets = unique_ptr<uint64_t[]>(new uint64_t[count]);
+	bool child_contiguous = true;
 	for (idx_t i = 0; i < count; i++) {
 		auto input_idx = list_data.sel->get_index(i);
 		if (list_validity.RowIsValid(input_idx)) {
@@ -186,17 +180,11 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 			if (input_list.offset != child_count) {
 				child_contiguous = false;
 			}
-			append_offsets[i].offset = start_offset + child_count;
-			append_offsets[i].length = input_list.length;
+			append_offsets[i] = start_offset + child_count + input_list.length;
 			child_count += input_list.length;
 		} else {
 			append_mask.SetInvalid(i);
-			if (i > 0) {
-				append_offsets[i].offset = append_offsets[i - 1].offset + append_offsets[i - 1].length;
-			} else {
-				append_offsets[i].offset = start_offset;
-			}
-			append_offsets[i].length = 0;
+			append_offsets[i] = start_offset + child_count;
 		}
 	}
 	auto &list_child = ListVector::GetEntry(vector);
@@ -218,27 +206,19 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 		D_ASSERT(current_count == child_count);
 		child_vector.Slice(list_child, child_sel, child_count);
 	}
-#ifdef DEBUG
-	D_ASSERT(append_offsets[0].offset == start_offset);
-	for (idx_t i = 1; i < count; i++) {
-		D_ASSERT(append_offsets[i].offset == append_offsets[i - 1].offset + append_offsets[i - 1].length);
-	}
-	D_ASSERT(append_offsets[count - 1].offset + append_offsets[count - 1].length - append_offsets[0].offset ==
-	         child_count);
-#endif
 
 	UnifiedVectorFormat vdata;
-	vdata.validity = append_mask;
 	vdata.sel = FlatVector::IncrementalSelectionVector();
 	vdata.data = (data_ptr_t)append_offsets.get();
 
 	// append the list offsets
 	ColumnData::AppendData(stats, state, vdata, count);
 	// append the validity data
-	validity.AppendData(*stats.validity_stats, state.child_appends[0], vdata, count);
+	vdata.validity = append_mask;
+	validity.AppendData(stats, state.child_appends[0], vdata, count);
 	// append the child vector
 	if (child_count > 0) {
-		child_column->Append(*stats.child_stats, state.child_appends[1], child_vector, child_count);
+		child_column->Append(ListStats::GetChildStats(stats), state.child_appends[1], child_vector, child_count);
 	}
 }
 
@@ -248,8 +228,8 @@ void ListColumnData::RevertAppend(row_t start_row) {
 	auto column_count = GetMaxEntry();
 	if (column_count > start) {
 		// revert append in the child column
-		auto list_entry = FetchListEntry(column_count - 1);
-		child_column->RevertAppend(list_entry.offset + list_entry.length);
+		auto list_offset = FetchListOffset(column_count - 1);
+		child_column->RevertAppend(list_offset);
 	}
 }
 
@@ -281,19 +261,18 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 		auto child_state = make_unique<ColumnFetchState>();
 		state.child_states.push_back(std::move(child_state));
 	}
-	// fetch the list_entry_t and the validity mask for that list
-	auto segment = (ColumnSegment *)data.GetSegment(row_id);
 
 	// now perform the fetch within the segment
-	segment->FetchRow(state, row_id, result, result_idx);
+	auto start_offset = idx_t(row_id) == this->start ? 0 : FetchListOffset(row_id - 1);
+	auto end_offset = FetchListOffset(row_id);
 	validity.FetchRow(transaction, *state.child_states[0], row_id, result, result_idx);
 
 	auto &validity = FlatVector::Validity(result);
 	auto list_data = FlatVector::GetData<list_entry_t>(result);
 	auto &list_entry = list_data[result_idx];
-	auto original_offset = list_entry.offset;
 	// set the list entry offset to the size of the current list
 	list_entry.offset = ListVector::GetListSize(result);
+	list_entry.length = end_offset - start_offset;
 	if (!validity.RowIsValid(result_idx)) {
 		// the list is NULL! no need to fetch the child
 		D_ASSERT(list_entry.length == 0);
@@ -307,7 +286,8 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 		auto &child_type = ListType::GetChildType(result.GetType());
 		Vector child_scan(child_type, child_scan_count);
 		// seek the scan towards the specified position and read [length] entries
-		child_column->InitializeScanWithOffset(*child_state, start + original_offset);
+		child_state->Initialize(child_type);
+		child_column->InitializeScanWithOffset(*child_state, start + start_offset);
 		D_ASSERT(child_type.InternalType() == PhysicalType::STRUCT ||
 		         child_state->row_index + child_scan_count - this->start <= child_column->GetMaxEntry());
 		child_column->ScanCount(*child_state, child_scan, child_scan_count);
@@ -324,7 +304,7 @@ void ListColumnData::CommitDropColumn() {
 struct ListColumnCheckpointState : public ColumnCheckpointState {
 	ListColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, PartialBlockManager &partial_block_manager)
 	    : ColumnCheckpointState(row_group, column_data, partial_block_manager) {
-		global_stats = make_unique<ListStatistics>(column_data.type);
+		global_stats = ListStats::CreateEmpty(column_data.type).ToUnique();
 	}
 
 	unique_ptr<ColumnCheckpointState> validity_state;
@@ -333,10 +313,8 @@ struct ListColumnCheckpointState : public ColumnCheckpointState {
 public:
 	unique_ptr<BaseStatistics> GetStatistics() override {
 		auto stats = global_stats->Copy();
-		auto &list_stats = (ListStatistics &)*stats;
-		stats->validity_stats = validity_state->GetStatistics();
-		list_stats.child_stats = child_state->GetStatistics();
-		return stats;
+		ListStats::SetChildStats(stats, child_state->GetStatistics());
+		return stats.ToUnique();
 	}
 
 	void WriteDataPointers(RowGroupWriter &writer) override {
@@ -376,6 +354,7 @@ void ListColumnData::DeserializeColumn(Deserializer &source) {
 }
 
 void ListColumnData::GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result) {
+	ColumnData::GetStorageInfo(row_group_index, col_path, result);
 	col_path.push_back(0);
 	validity.GetStorageInfo(row_group_index, col_path, result);
 	col_path.back() = 1;

package/src/duckdb/src/storage/table/persistent_table_data.cpp

@@ -3,7 +3,8 @@
 
 namespace duckdb {
 
-PersistentTableData::PersistentTableData(idx_t column_count) {
+PersistentTableData::PersistentTableData(idx_t column_count)
+    : total_rows(0), row_group_count(0), block_id(INVALID_BLOCK), offset(0) {
 }
 
 PersistentTableData::~PersistentTableData() {