duckdb 0.10.1 → 0.10.2-dev3.0

package/src/duckdb/src/transaction/commit_state.cpp

@@ -204,8 +204,15 @@ void CommitState::WriteDelete(DeleteInfo &info) {
 		delete_chunk->Initialize(Allocator::DefaultAllocator(), delete_types);
 	}
 	auto rows = FlatVector::GetData<row_t>(delete_chunk->data[0]);
-	for (idx_t i = 0; i < info.count; i++) {
-		rows[i] = info.base_row + info.rows[i];
+	if (info.is_consecutive) {
+		for (idx_t i = 0; i < info.count; i++) {
+			rows[i] = info.base_row + i;
+		}
+	} else {
+		auto delete_rows = info.GetRows();
+		for (idx_t i = 0; i < info.count; i++) {
+			rows[i] = info.base_row + delete_rows[i];
+		}
 	}
 	delete_chunk->SetCardinality(info.count);
 	log->WriteDelete(*delete_chunk);
@@ -310,7 +317,7 @@ void CommitState::CommitEntry(UndoFlags type, data_ptr_t data) {
 			WriteDelete(*info);
 		}
 		// mark the tuples as committed
-		info->version_info->CommitDelete(info->vector_idx, commit_id, info->rows, info->count);
+		info->version_info->CommitDelete(info->vector_idx, commit_id, *info);
 		break;
 	}
 	case UndoFlags::UPDATE_TUPLE: {
@@ -351,7 +358,7 @@ void CommitState::RevertCommit(UndoFlags type, data_ptr_t data) {
 		auto info = reinterpret_cast<DeleteInfo *>(data);
 		info->table->info->cardinality += info->count;
 		// revert the commit by writing the (uncommitted) transaction_id back into the version info
-		info->version_info->CommitDelete(info->vector_idx, transaction_id, info->rows, info->count);
+		info->version_info->CommitDelete(info->vector_idx, transaction_id, *info);
 		break;
 	}
 	case UndoFlags::UPDATE_TUPLE: {

package/src/duckdb/src/transaction/duck_transaction.cpp

@@ -73,14 +73,34 @@ void DuckTransaction::PushCatalogEntry(CatalogEntry &entry, data_ptr_t extra_dat
 
 void DuckTransaction::PushDelete(DataTable &table, RowVersionManager &info, idx_t vector_idx, row_t rows[], idx_t count,
                                  idx_t base_row) {
-	auto delete_info = reinterpret_cast<DeleteInfo *>(
-	    undo_buffer.CreateEntry(UndoFlags::DELETE_TUPLE, sizeof(DeleteInfo) + sizeof(row_t) * count));
+	bool is_consecutive = true;
+	// check if the rows are consecutive
+	for (idx_t i = 0; i < count; i++) {
+		if (rows[i] != row_t(i)) {
+			is_consecutive = false;
+			break;
+		}
+	}
+	idx_t alloc_size = sizeof(DeleteInfo);
+	if (!is_consecutive) {
+		// if rows are not consecutive we need to allocate row identifiers
+		alloc_size += sizeof(uint16_t) * count;
+	}
+
+	auto delete_info = reinterpret_cast<DeleteInfo *>(undo_buffer.CreateEntry(UndoFlags::DELETE_TUPLE, alloc_size));
 	delete_info->version_info = &info;
 	delete_info->vector_idx = vector_idx;
 	delete_info->table = &table;
 	delete_info->count = count;
 	delete_info->base_row = base_row;
-	memcpy(delete_info->rows, rows, sizeof(row_t) * count);
+	delete_info->is_consecutive = is_consecutive;
+	if (!is_consecutive) {
+		// if rows are not consecutive
+		auto delete_rows = delete_info->GetRows();
+		for (idx_t i = 0; i < count; i++) {
+			delete_rows[i] = NumericCast<uint16_t>(rows[i]);
+		}
+	}
 }
 
 void DuckTransaction::PushAppend(DataTable &table, idx_t start_row, idx_t row_count) {

package/src/duckdb/src/transaction/rollback_state.cpp

@@ -31,7 +31,7 @@ void RollbackState::RollbackEntry(UndoFlags type, data_ptr_t data) {
 	case UndoFlags::DELETE_TUPLE: {
 		auto info = reinterpret_cast<DeleteInfo *>(data);
 		// reset the deleted flag on rollback
-		info->version_info->CommitDelete(info->vector_idx, NOT_DELETED_ID, info->rows, info->count);
+		info->version_info->CommitDelete(info->vector_idx, NOT_DELETED_ID, *info);
 		break;
 	}
 	case UndoFlags::UPDATE_TUPLE: {

package/src/duckdb/src/transaction/transaction_context.cpp

@@ -18,7 +18,7 @@ TransactionContext::~TransactionContext() {
 	if (current_transaction) {
 		try {
 			Rollback();
-		} catch (...) {
+		} catch (...) { // NOLINT
 		}
 	}
 }

package/src/duckdb/src/transaction/undo_buffer.cpp

@@ -142,7 +142,9 @@ void UndoBuffer::Cleanup() {
 	// possibly vacuum indexes
 	for (const auto &table : state.indexed_tables) {
 		table.second->info->indexes.Scan([&](Index &index) {
-			index.Vacuum();
+			if (!index.IsUnknown()) {
+				index.Vacuum();
+			}
 			return false;
 		});
 	}

package/src/duckdb/third_party/fmt/include/fmt/core.h

@@ -353,12 +353,7 @@ template <typename Char> class basic_string_view {
 using string_view = basic_string_view<char>;
 using wstring_view = basic_string_view<wchar_t>;
 
-// A UTF-8 code unit type.
-#if FMT_HAS_FEATURE(__cpp_char8_t)
-typedef char8_t fmt_char8_t;
-#else
 typedef char fmt_char8_t;
-#endif
 
 /** Specifies if ``T`` is a character type. Can be specialized by users. */
 template <typename T> struct is_char : std::false_type {};

package/src/duckdb/third_party/fsst/fsst.h

@@ -48,7 +48,7 @@
 #ifndef FSST_INCLUDED_H
 #define FSST_INCLUDED_H
 
-#ifdef _MSC_VER
+#if defined(_MSC_VER) && !defined(__clang__)
 #define __restrict__ 
 #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
 #define __ORDER_LITTLE_ENDIAN__ 2

package/src/duckdb/third_party/fsst/libfsst.cpp

@@ -241,140 +241,6 @@ SymbolTable *buildSymbolTable(Counters& counters, vector<u8*> line, size_t len[]
 	return bestTable;
 }
 
-static inline size_t compressSIMD(SymbolTable &symbolTable, u8* symbolBase, size_t nlines, size_t len[], u8* line[], size_t size, u8* dst, size_t lenOut[], u8* strOut[], int unroll) {
-	size_t curLine = 0, inOff = 0, outOff = 0, batchPos = 0, empty = 0, budget = size;
-	u8 *lim = dst + size, *codeBase = symbolBase + (1<<18); // 512KB temp space for compressing 512 strings
-	SIMDjob input[512];  // combined offsets of input strings (cur,end), and string #id (pos) and output (dst) pointer
-	SIMDjob output[512]; // output are (pos:9,dst:19) end pointers (compute compressed length from this)
-	size_t jobLine[512]; // for which line in the input sequence was this job (needed because we may split a line into multiple jobs)
-
-	while (curLine < nlines && outOff <= (1<<19)) {
-		size_t prevLine = curLine, chunk, curOff = 0;
-
-		// bail out if the output buffer cannot hold the compressed next string fully
-		if (((len[curLine]-curOff)*2 + 7) > budget) break; // see below for the +7
-		else budget -= (len[curLine]-curOff)*2;
-
-		strOut[curLine] = (u8*) 0;
-		lenOut[curLine] = 0;
-
-		do {
-			do {
-				chunk = len[curLine] - curOff;
-				if (chunk > 511) {
-					chunk = 511; // large strings need to be chopped up into segments of 511 bytes
-				}
-				// create a job in this batch
-				SIMDjob job;
-				job.cur = inOff;
-				job.end = job.cur + chunk;
-				job.pos = batchPos;
-				job.out = outOff;
-
-				// worst case estimate for compressed size (+7 is for the scatter that writes extra 7 zeros)
-				outOff += 7 + 2*(size_t)(job.end - job.cur); // note, total size needed is 512*(511*2+7) bytes.
-				if (outOff > (1<<19)) break; // simdbuf may get full, stop before this chunk
-
-				// register job in this batch
-				input[batchPos] = job;
-				jobLine[batchPos] = curLine;
-
-				if (chunk == 0) {
-					empty++; // detect empty chunks -- SIMD code cannot handle empty strings, so they need to be filtered out
-				} else {
-					// copy string chunk into temp buffer
-					memcpy(symbolBase + inOff, line[curLine] + curOff, chunk);
-					inOff += chunk;
-					curOff += chunk;
-					symbolBase[inOff++] = (u8) symbolTable.terminator; // write an extra char at the end that will not be encoded
-				}
-				if (++batchPos == 512) break;
-			} while(curOff < len[curLine]);
-
-			if ((batchPos == 512) || (outOff > (1<<19)) || (++curLine >= nlines)) { // cannot accumulate more?
-				if (batchPos-empty >= 32) { // if we have enough work, fire off fsst_compressAVX512 (32 is due to max 4x8 unrolling)
-					// radix-sort jobs on length (longest string first)
-					// -- this provides best load balancing and allows to skip empty jobs at the end
-					u16 sortpos[513];
-					memset(sortpos, 0, sizeof(sortpos));
-
-					// calculate length histo
-					for(size_t i=0; i<batchPos; i++) {
-						size_t len = input[i].end - input[i].cur;
-						sortpos[512UL - len]++;
-					}
-					// calculate running sum
-					for(size_t i=1; i<=512; i++)
-						sortpos[i] += sortpos[i-1];
-
-					// move jobs to their final destination
-					SIMDjob inputOrdered[512];
-					for(size_t i=0; i<batchPos; i++) {
-						size_t len = input[i].end - input[i].cur;
-						size_t pos = sortpos[511UL - len]++;
-						inputOrdered[pos] = input[i];
-					}
-					// finally.. SIMD compress max 256KB of simdbuf into (max) 512KB of simdbuf (but presumably much less..)
-					for(size_t done = duckdb_fsst_compressAVX512(symbolTable, codeBase, symbolBase, inputOrdered, output, batchPos-empty, unroll);
-					     done < batchPos; done++) output[done] = inputOrdered[done];
-				} else {
-					memcpy(output, input, batchPos*sizeof(SIMDjob));
-				}
-
-				// finish encoding (unfinished strings in process, plus the few last strings not yet processed)
-				for(size_t i=0; i<batchPos; i++) {
-					SIMDjob job = output[i];
-					if (job.cur < job.end) { // finish encoding this string with scalar code
-						u8* cur = symbolBase + job.cur;
-						u8* end = symbolBase + job.end;
-						u8* out = codeBase + job.out;
-						while (cur < end) {
-							u64 word = fsst_unaligned_load(cur);
-							size_t code = symbolTable.shortCodes[word & 0xFFFF];
-							size_t pos = word & 0xFFFFFF;
-							size_t idx = FSST_HASH(pos)&(symbolTable.hashTabSize-1);
-							Symbol s = symbolTable.hashTab[idx];
-							out[1] = (u8) word; // speculatively write out escaped byte
-							word &= (0xFFFFFFFFFFFFFFFF >> (u8) s.icl);
-							if ((s.icl < FSST_ICL_FREE) && s.val.num == word) {
-								*out++ = (u8) s.code(); cur += s.length();
-							} else {
-								// could be a 2-byte or 1-byte code, or miss
-								// handle everything with predication
-								*out = (u8) code;
-								out += 1+((code&FSST_CODE_BASE)>>8);
-								cur += (code>>FSST_LEN_BITS);
-							}
-						}
-						job.out = out - codeBase;
-					}
-					// postprocess job info
-					job.cur = 0;
-					job.end = job.out - input[job.pos].out; // misuse .end field as compressed size
-					job.out = input[job.pos].out; // reset offset to start of encoded string
-					input[job.pos] = job;
-				}
-
-				// copy out the result data
-				for(size_t i=0; i<batchPos; i++) {
-					size_t lineNr = jobLine[i]; // the sort must be order-preserving, as we concatenate results string in order
-					size_t sz = input[i].end; // had stored compressed lengths here
-					if (!strOut[lineNr]) strOut[lineNr] = dst; // first segment will be the strOut pointer
-					lenOut[lineNr] += sz; // add segment (lenOut starts at 0 for this reason)
-					memcpy(dst, codeBase+input[i].out, sz);
-					dst += sz;
-				}
-
-				// go for the next batch of 512 chunks
-				inOff = outOff = batchPos = empty = 0;
-				budget = (size_t) (lim - dst);
-			}
-		} while (curLine == prevLine && outOff <= (1<<19));
-	}
-	return curLine;
-}
-
-
 // optimized adaptive *scalar* compression method
 static inline size_t compressBulk(SymbolTable &symbolTable, size_t nlines, size_t lenIn[], u8* strIn[], size_t size, u8* out, size_t lenOut[], u8* strOut[], bool noSuffixOpt, bool avoidBranch) {
 	u8 *cur = NULL, *end =  NULL, *lim = out + size;
@@ -592,12 +458,7 @@ extern "C" u32 duckdb_fsst_import(duckdb_fsst_decoder_t *decoder, u8 *buf) {
 }
 
 // runtime check for simd
-inline size_t _compressImpl(Encoder *e, size_t nlines, size_t lenIn[], u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd) {
-#ifndef NONOPT_FSST
-	if (simd && duckdb_fsst_hasAVX512())
-		return compressSIMD(*e->symbolTable, e->simdbuf, nlines, lenIn, strIn, size, output, lenOut, strOut, simd);
-#endif
-	(void) simd;
+inline size_t _compressImpl(Encoder *e, size_t nlines, size_t lenIn[], u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int) {
 	return compressBulk(*e->symbolTable, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch);
 }
 size_t compressImpl(Encoder *e, size_t nlines, size_t lenIn[], u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd) {

package/src/duckdb/third_party/fsst/libfsst.hpp

@@ -432,19 +432,6 @@ struct SIMDjob {
    u64 out:19,pos:9,end:18,cur:18; // cur/end is input offsets (2^18=256KB), out is output offset (2^19=512KB)  
 };
 
-extern bool 
-duckdb_fsst_hasAVX512(); // runtime check for avx512 capability
-
-extern size_t 
-duckdb_fsst_compressAVX512(
-   SymbolTable &symbolTable, 
-   u8* codeBase,    // IN: base address for codes, i.e. compression output (points to simdbuf+256KB)
-   u8* symbolBase,  // IN: base address for string bytes, i.e. compression input (points to simdbuf)
-   SIMDjob* input,  // IN: input array (size n) with job information: what to encode, where to store it.
-   SIMDjob* output, // OUT: output array (size n) with job information: how much got encoded, end output pointer.
-   size_t n,         // IN: size of arrays input and output (should be max 512)
-   size_t unroll);   // IN: degree of SIMD unrolling
-
 // C++ fsst-compress function with some more control of how the compression happens (algorithm flavor, simd unroll degree)
 size_t compressImpl(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd);
 size_t compressAuto(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], int simd);

package/src/duckdb/third_party/hyperloglog/hyperloglog.hpp

@@ -13,6 +13,8 @@
 
 namespace duckdb_hll {
 
+// NOLINTBEGIN
+
 /* Error codes */
 #define HLL_C_OK  0
 #define HLL_C_ERR -1
@@ -40,6 +42,8 @@ uint64_t get_size();
 
 uint64_t MurmurHash64A(const void *key, int len, unsigned int seed);
 
+// NOLINTEND
+
 } // namespace duckdb_hll
 
 namespace duckdb {

package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp

@@ -51,6 +51,13 @@ typedef enum PGSortByDir {
 	SORTBY_USING /* not allowed in CREATE INDEX ... */
 } PGSortByDir;
 
+/* PGFuncCall options RESPECT/IGNORE NULLS */
+typedef enum PGIgnoreNulls {
+	PG_DEFAULT_NULLS,
+	PG_RESPECT_NULLS,
+	PG_IGNORE_NULLS
+} PGIgnoreNulls;
+
 typedef enum PGSortByNulls { PG_SORTBY_NULLS_DEFAULT, PG_SORTBY_NULLS_FIRST, PG_SORTBY_NULLS_LAST } PGSortByNulls;
 
 /*****************************************************************************
@@ -292,7 +299,7 @@ typedef struct PGFuncCall {
 	bool agg_within_group;    /* ORDER BY appeared in WITHIN GROUP */
 	bool agg_star;            /* argument was really '*' */
 	bool agg_distinct;        /* arguments were labeled DISTINCT */
-	bool agg_ignore_nulls;    /* arguments were labeled IGNORE NULLS */
+	PGIgnoreNulls agg_ignore_nulls; /* arguments were labeled IGNORE NULLS */
 	bool func_variadic;       /* last argument was labeled VARIADIC */
 	struct PGWindowDef *over; /* OVER clause, if any */
 	int location;             /* token location, or -1 if unknown */

package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp

@@ -1089,6 +1089,7 @@ typedef union YYSTYPE
 	PGOverridingKind       override;
 	PGSortByDir            sortorder;
 	PGSortByNulls          nullorder;
+	PGIgnoreNulls          ignorenulls;
 	PGConstrType           constr;
 	PGLockClauseStrength lockstrength;
 	PGLockWaitPolicy lockwaitpolicy;
@@ -1097,7 +1098,7 @@ typedef union YYSTYPE
 	PGInsertColumnOrder bynameorposition;
 }
 /* Line 1529 of yacc.c.  */
-#line 1101 "third_party/libpg_query/grammar/grammar_out.hpp"
+#line 1102 "third_party/libpg_query/grammar/grammar_out.hpp"
 	YYSTYPE;
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */
 # define YYSTYPE_IS_DECLARED 1