duckdb 0.7.2-dev16.0 → 0.7.2-dev225.0

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 43;
+const uint64_t VERSION_NUMBER = 44;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/table/column_data.cpp

@@ -466,8 +466,11 @@ void ColumnData::GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, T
 		column_info.segment_start = segment->start;
 		column_info.segment_count = segment->count;
 		column_info.compression_type = CompressionTypeToString(segment->function->type);
-		column_info.segment_stats =
-		    segment->stats.statistics ? segment->stats.statistics->ToString() : string("No Stats");
+		if (!segment->stats.statistics || type.id() == LogicalTypeId::LIST) {
+			column_info.segment_stats = string("No Stats");
+		} else {
+			column_info.segment_stats = segment->stats.statistics->ToString();
+		}
 		column_info.has_updates = updates ? true : false;
 		// persistent
 		// block_id

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -39,15 +39,14 @@ void ListColumnData::InitializeScan(ColumnScanState &state) {
 	state.child_states.push_back(std::move(child_state));
 }
 
-list_entry_t ListColumnData::FetchListEntry(idx_t row_idx) {
+uint64_t ListColumnData::FetchListOffset(idx_t row_idx) {
 	auto segment = (ColumnSegment *)data.GetSegment(row_idx);
 	ColumnFetchState fetch_state;
 	Vector result(type, 1);
 	segment->FetchRow(fetch_state, row_idx, result, 0);
 
 	// initialize the child scan with the required offset
-	auto list_data = FlatVector::GetData<list_entry_t>(result);
-	return list_data[0];
+	return FlatVector::GetData<uint64_t>(result)[0];
 }
 
 void ListColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) {
@@ -63,8 +62,7 @@ void ListColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_
 	state.child_states.push_back(std::move(validity_state));
 
 	// we need to read the list at position row_idx to get the correct row offset of the child
-	auto list_entry = FetchListEntry(row_idx);
-	auto child_offset = list_entry.offset;
+	auto child_offset = row_idx == start ? 0 : FetchListOffset(row_idx - 1);
 
 	D_ASSERT(child_offset <= child_column->GetMaxEntry());
 	ColumnScanState child_state;
@@ -89,26 +87,26 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	// updates not supported for lists
 	D_ASSERT(!updates);
 
-	idx_t scan_count = ScanVector(state, result, count);
+	Vector offset_vector(LogicalType::UBIGINT, count);
+	idx_t scan_count = ScanVector(state, offset_vector, count);
 	D_ASSERT(scan_count > 0);
 	validity.ScanCount(state.child_states[0], result, count);
 
-	auto data = FlatVector::GetData<list_entry_t>(result);
-	auto first_entry = data[0];
+	auto data = FlatVector::GetData<uint64_t>(offset_vector);
 	auto last_entry = data[scan_count - 1];
 
-#ifdef DEBUG
-	for (idx_t i = 1; i < scan_count; i++) {
-		D_ASSERT(data[i].offset == data[i - 1].offset + data[i - 1].length);
-	}
-#endif
 	// shift all offsets so they are 0 at the first entry
+	auto result_data = FlatVector::GetData<list_entry_t>(result);
+	auto base_offset = state.last_offset;
+	idx_t current_offset = 0;
 	for (idx_t i = 0; i < scan_count; i++) {
-		data[i].offset -= first_entry.offset;
+		result_data[i].offset = current_offset;
+		result_data[i].length = data[i] - current_offset - base_offset;
+		current_offset += result_data[i].length;
 	}
 
-	D_ASSERT(last_entry.offset >= first_entry.offset);
-	idx_t child_scan_count = last_entry.offset + last_entry.length - first_entry.offset;
+	D_ASSERT(last_entry >= base_offset);
+	idx_t child_scan_count = last_entry - base_offset;
 	ListVector::Reserve(result, child_scan_count);
 
 	if (child_scan_count > 0) {
@@ -118,6 +116,7 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 		             child_column->start + child_column->GetMaxEntry());
 		child_column->ScanCount(state.child_states[1], child_entry, child_scan_count);
 	}
+	state.last_offset = last_entry;
 
 	ListVector::SetListSize(result, child_scan_count);
 	return scan_count;
@@ -130,19 +129,19 @@ void ListColumnData::Skip(ColumnScanState &state, idx_t count) {
 	// we need to read the list entries/offsets to figure out how much to skip
 	// note that we only need to read the first and last entry
 	// however, let's just read all "count" entries for now
-	auto data = unique_ptr<list_entry_t[]>(new list_entry_t[count]);
-	Vector result(type, (data_ptr_t)data.get());
+	Vector result(LogicalType::UBIGINT, count);
 	idx_t scan_count = ScanVector(state, result, count);
 	if (scan_count == 0) {
 		return;
 	}
 
-	auto &first_entry = data[0];
-	auto &last_entry = data[scan_count - 1];
-	idx_t child_scan_count = last_entry.offset + last_entry.length - first_entry.offset;
+	auto data = FlatVector::GetData<uint64_t>(result);
+	auto last_entry = data[scan_count - 1];
+	idx_t child_scan_count = last_entry - state.last_offset;
 	if (child_scan_count == 0) {
 		return;
 	}
+	state.last_offset = last_entry;
 
 	// skip the child state forward by the child_scan_count
 	child_column->Skip(state.child_states[1], child_scan_count);
@@ -177,8 +176,8 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 	idx_t child_count = 0;
 
 	ValidityMask append_mask(count);
-	auto append_offsets = unique_ptr<list_entry_t[]>(new list_entry_t[count]);
-	bool child_contiguous = false;
+	auto append_offsets = unique_ptr<uint64_t[]>(new uint64_t[count]);
+	bool child_contiguous = true;
 	for (idx_t i = 0; i < count; i++) {
 		auto input_idx = list_data.sel->get_index(i);
 		if (list_validity.RowIsValid(input_idx)) {
@@ -186,17 +185,11 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 			if (input_list.offset != child_count) {
 				child_contiguous = false;
 			}
-			append_offsets[i].offset = start_offset + child_count;
-			append_offsets[i].length = input_list.length;
+			append_offsets[i] = start_offset + child_count + input_list.length;
 			child_count += input_list.length;
 		} else {
 			append_mask.SetInvalid(i);
-			if (i > 0) {
-				append_offsets[i].offset = append_offsets[i - 1].offset + append_offsets[i - 1].length;
-			} else {
-				append_offsets[i].offset = start_offset;
-			}
-			append_offsets[i].length = 0;
+			append_offsets[i] = start_offset + child_count;
 		}
 	}
 	auto &list_child = ListVector::GetEntry(vector);
@@ -218,23 +211,15 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 		D_ASSERT(current_count == child_count);
 		child_vector.Slice(list_child, child_sel, child_count);
 	}
-#ifdef DEBUG
-	D_ASSERT(append_offsets[0].offset == start_offset);
-	for (idx_t i = 1; i < count; i++) {
-		D_ASSERT(append_offsets[i].offset == append_offsets[i - 1].offset + append_offsets[i - 1].length);
-	}
-	D_ASSERT(append_offsets[count - 1].offset + append_offsets[count - 1].length - append_offsets[0].offset ==
-	         child_count);
-#endif
 
 	UnifiedVectorFormat vdata;
-	vdata.validity = append_mask;
 	vdata.sel = FlatVector::IncrementalSelectionVector();
 	vdata.data = (data_ptr_t)append_offsets.get();
 
 	// append the list offsets
 	ColumnData::AppendData(stats, state, vdata, count);
 	// append the validity data
+	vdata.validity = append_mask;
 	validity.AppendData(*stats.validity_stats, state.child_appends[0], vdata, count);
 	// append the child vector
 	if (child_count > 0) {
@@ -248,8 +233,8 @@ void ListColumnData::RevertAppend(row_t start_row) {
 	auto column_count = GetMaxEntry();
 	if (column_count > start) {
 		// revert append in the child column
-		auto list_entry = FetchListEntry(column_count - 1);
-		child_column->RevertAppend(list_entry.offset + list_entry.length);
+		auto list_offset = FetchListOffset(column_count - 1);
+		child_column->RevertAppend(list_offset);
 	}
 }
 
@@ -281,19 +266,18 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 		auto child_state = make_unique<ColumnFetchState>();
 		state.child_states.push_back(std::move(child_state));
 	}
-	// fetch the list_entry_t and the validity mask for that list
-	auto segment = (ColumnSegment *)data.GetSegment(row_id);
 
 	// now perform the fetch within the segment
-	segment->FetchRow(state, row_id, result, result_idx);
+	auto start_offset = idx_t(row_id) == this->start ? 0 : FetchListOffset(row_id - 1);
+	auto end_offset = FetchListOffset(row_id);
 	validity.FetchRow(transaction, *state.child_states[0], row_id, result, result_idx);
 
 	auto &validity = FlatVector::Validity(result);
 	auto list_data = FlatVector::GetData<list_entry_t>(result);
 	auto &list_entry = list_data[result_idx];
-	auto original_offset = list_entry.offset;
 	// set the list entry offset to the size of the current list
 	list_entry.offset = ListVector::GetListSize(result);
+	list_entry.length = end_offset - start_offset;
 	if (!validity.RowIsValid(result_idx)) {
 		// the list is NULL! no need to fetch the child
 		D_ASSERT(list_entry.length == 0);
@@ -307,7 +291,7 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 		auto &child_type = ListType::GetChildType(result.GetType());
 		Vector child_scan(child_type, child_scan_count);
 		// seek the scan towards the specified position and read [length] entries
-		child_column->InitializeScanWithOffset(*child_state, start + original_offset);
+		child_column->InitializeScanWithOffset(*child_state, start + start_offset);
 		D_ASSERT(child_type.InternalType() == PhysicalType::STRUCT ||
 		         child_state->row_index + child_scan_count - this->start <= child_column->GetMaxEntry());
 		child_column->ScanCount(*child_state, child_scan, child_scan_count);
@@ -376,6 +360,7 @@ void ListColumnData::DeserializeColumn(Deserializer &source) {
 }
 
 void ListColumnData::GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result) {
+	ColumnData::GetStorageInfo(row_group_index, col_path, result);
 	col_path.push_back(0);
 	validity.GetStorageInfo(row_group_index, col_path, result);
 	col_path.back() = 1;

package/src/duckdb/third_party/libpg_query/include/nodes/nodes.hpp

@@ -194,6 +194,9 @@ typedef enum PGNodeTag {
 	T_PGOnConflictExpr,
 	T_PGIntoClause,
 	T_PGLambdaFunction,
+	T_PGPivotExpr,
+	T_PGPivot,
+	T_PGPivotStmt,
 
 	/*
 	 * TAGS FOR EXPRESSION STATE NODES (execnodes.h)

package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp

@@ -1162,6 +1162,37 @@ typedef struct PGUpdateStmt {
 	PGWithClause *withClause; /* WITH clause */
 } PGUpdateStmt;
 
+/* ----------------------
+ *		Pivot Expression
+ * ----------------------
+ */
+typedef struct PGPivot {
+	PGNodeTag type;
+	PGList *pivot_columns;  /* The column names to pivot on */
+	PGList *pivot_value;    /* The set of pivot values */
+	char *pivot_enum;       /* The enum to fetch the unique values from */
+} PGPivot;
+
+typedef struct PGPivotExpr {
+	PGNodeTag type;
+	PGNode *source;      /* the source subtree */
+	PGList *aggrs;       /* The aggregations to pivot over (PIVOT only) */
+	PGList *unpivots;    /* The names to unpivot over (UNPIVOT only) */
+	PGList *pivots;      /* The set of pivot values */
+	PGList *groups;      /* The set of groups to pivot over (if any) */
+	PGAlias *alias;      /* table alias & optional column aliases */
+	bool include_nulls;  /* Whether or not to include NULL values (UNPIVOT only */
+} PGPivotExpr;
+
+typedef struct PGPivotStmt {
+	PGNodeTag type;
+	PGNode *source;      /* The source to pivot */
+	PGList *aggrs;       /* The aggregations to pivot over (PIVOT only) */
+	PGList *unpivots;    /* The names to unpivot over (UNPIVOT only) */
+	PGList *columns;     /* The set of columns to pivot over */
+	PGList *groups;      /* The set of groups to pivot over (if any) */
+} PGPivotStmt;
+
 /* ----------------------
  *		Select Statement
  *
@@ -1204,6 +1235,9 @@ typedef struct PGSelectStmt {
 	 */
 	PGList *valuesLists; /* untransformed list of expression lists */
 
+	/* When representing a pivot statement, all values are NULL besides the pivot field */
+	PGPivotStmt *pivot;       /* PIVOT statement */
+
 	/*
 	 * These fields are used in both "leaf" SelectStmts and upper-level
 	 * SelectStmts.
@@ -2107,5 +2141,4 @@ typedef struct PGUseStmt {
 } PGUseStmt;
 
 
-
 }