duckdb 0.10.2-dev0.0 → 0.10.2-dev3.0

package/src/duckdb/src/execution/expression_executor.cpp

@@ -87,9 +87,7 @@ void ExpressionExecutor::ExecuteExpression(DataChunk &input, Vector &result) {
 idx_t ExpressionExecutor::SelectExpression(DataChunk &input, SelectionVector &sel) {
 	D_ASSERT(expressions.size() == 1);
 	SetChunk(&input);
-	states[0]->profiler.BeginSample();
 	idx_t selected_tuples = Select(*expressions[0], states[0]->root_state.get(), nullptr, input.size(), &sel, nullptr);
-	states[0]->profiler.EndSample(NumericCast<int>(chunk ? chunk->size() : 0));
 	return selected_tuples;
 }
 
@@ -101,9 +99,7 @@ void ExpressionExecutor::ExecuteExpression(Vector &result) {
 void ExpressionExecutor::ExecuteExpression(idx_t expr_idx, Vector &result) {
 	D_ASSERT(expr_idx < expressions.size());
 	D_ASSERT(result.GetType().id() == expressions[expr_idx]->return_type.id());
-	states[expr_idx]->profiler.BeginSample();
 	Execute(*expressions[expr_idx], states[expr_idx]->root_state.get(), nullptr, chunk ? chunk->size() : 1, result);
-	states[expr_idx]->profiler.EndSample(NumericCast<int>(chunk ? chunk->size() : 0));
 }
 
 Value ExpressionExecutor::EvaluateScalar(ClientContext &context, const Expression &expr, bool allow_unfoldable) {

package/src/duckdb/src/execution/expression_executor_state.cpp

@@ -40,7 +40,7 @@ ClientContext &ExpressionState::GetContext() {
 ExpressionState::ExpressionState(const Expression &expr, ExpressionExecutorState &root) : expr(expr), root(root) {
 }
 
-ExpressionExecutorState::ExpressionExecutorState() : profiler() {
+ExpressionExecutorState::ExpressionExecutorState() {
 }
 
 void ExpressionState::Verify(ExpressionExecutorState &root_executor) {

package/src/duckdb/src/execution/index/art/art.cpp

@@ -133,13 +133,13 @@ unique_ptr<IndexScanState> ART::TryInitializeScan(const Transaction &transaction
 	// match on a comparison type
 	matcher.expr_type = make_uniq<ComparisonExpressionTypeMatcher>();
 	// match on a constant comparison with the indexed expression
-	matcher.matchers.push_back(make_uniq<ExpressionEqualityMatcher>(const_cast<Expression &>(index_expr)));
+	matcher.matchers.push_back(make_uniq<ExpressionEqualityMatcher>(index_expr));
 	matcher.matchers.push_back(make_uniq<ConstantExpressionMatcher>());
 
 	matcher.policy = SetMatcher::Policy::UNORDERED;
 
 	vector<reference<Expression>> bindings;
-	if (matcher.Match(const_cast<Expression &>(filter_expr), bindings)) {
+	if (matcher.Match(const_cast<Expression &>(filter_expr), bindings)) { // NOLINT: Match does not alter the expr
 		// range or equality comparison with constant value
 		// we can use our index here
 		// bindings[0] = the expression

package/src/duckdb/src/execution/index/unknown_index.cpp

@@ -22,44 +22,44 @@ string UnknownIndex::GenerateErrorMessage() const {
 }
 
 ErrorData UnknownIndex::Append(IndexLock &, DataChunk &, Vector &) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 void UnknownIndex::VerifyAppend(DataChunk &) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 void UnknownIndex::VerifyAppend(DataChunk &, ConflictManager &) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 void UnknownIndex::CommitDrop(IndexLock &) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 void UnknownIndex::Delete(IndexLock &, DataChunk &, Vector &) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 ErrorData UnknownIndex::Insert(IndexLock &, DataChunk &, Vector &) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 IndexStorageInfo UnknownIndex::GetStorageInfo(bool) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 bool UnknownIndex::MergeIndexes(IndexLock &, Index &) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 void UnknownIndex::Vacuum(IndexLock &) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 idx_t UnknownIndex::GetInMemorySize(IndexLock &) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 void UnknownIndex::CheckConstraintsForChunk(DataChunk &, ConflictManager &) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 string UnknownIndex::VerifyAndToString(IndexLock &, bool) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 
 string UnknownIndex::GetConstraintViolationMessage(VerifyExistenceType, idx_t, DataChunk &) {
-	throw NotImplementedException(GenerateErrorMessage());
+	throw MissingExtensionException(GenerateErrorMessage());
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/join_hashtable.cpp

@@ -513,7 +513,7 @@ void ScanStructure::NextInnerJoin(DataChunk &keys, DataChunk &left, DataChunk &r
 				Store<bool>(true, ptrs[idx] + ht.tuple_size);
 			}
 		}
-		// for right semi join, just mark the entry as found and move on. Propogation happens later
+		// for right semi join, just mark the entry as found and move on. Propagation happens later
 		if (ht.join_type != JoinType::RIGHT_SEMI && ht.join_type != JoinType::RIGHT_ANTI) {
 			// matches were found
 			// construct the result

package/src/duckdb/src/execution/nested_loop_join/nested_loop_join_mark.cpp

@@ -41,7 +41,6 @@ static void TemplatedMarkJoin(Vector &left, Vector &right, idx_t lcount, idx_t r
 static void MarkJoinNested(Vector &left, Vector &right, idx_t lcount, idx_t rcount, bool found_match[],
                            ExpressionType comparison_type) {
 	Vector left_reference(left.GetType());
-	SelectionVector true_sel(rcount);
 	for (idx_t i = 0; i < lcount; i++) {
 		if (found_match[i]) {
 			continue;

package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp

@@ -5,7 +5,8 @@ namespace duckdb {
 
 CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
                      idx_t &global_csv_current_position, idx_t file_number_p)
-    : context(context), file_number(file_number_p), can_seek(file_handle.CanSeek()) {
+    : context(context), requested_size(buffer_size_p), file_number(file_number_p), can_seek(file_handle.CanSeek()),
+      is_pipe(file_handle.IsPipe()) {
 	AllocateBuffer(buffer_size_p);
 	auto buffer = Ptr();
 	actual_buffer_size = file_handle.Read(buffer, buffer_size_p);
@@ -19,8 +20,9 @@ CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle
 
 CSVBuffer::CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size,
                      idx_t global_csv_current_position, idx_t file_number_p, idx_t buffer_idx_p)
-    : context(context), global_csv_start(global_csv_current_position), file_number(file_number_p),
-      can_seek(file_handle.CanSeek()), buffer_idx(buffer_idx_p) {
+    : context(context), requested_size(buffer_size), global_csv_start(global_csv_current_position),
+      file_number(file_number_p), can_seek(file_handle.CanSeek()), is_pipe(file_handle.IsPipe()),
+      buffer_idx(buffer_idx_p) {
 	AllocateBuffer(buffer_size);
 	auto buffer = handle.Ptr();
 	actual_buffer_size = file_handle.Read(handle.Ptr(), buffer_size);
@@ -50,7 +52,7 @@ shared_ptr<CSVBuffer> CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_s
 
 void CSVBuffer::AllocateBuffer(idx_t buffer_size) {
 	auto &buffer_manager = BufferManager::GetBufferManager(context);
-	bool can_destroy = can_seek;
+	bool can_destroy = !is_pipe;
 	handle = buffer_manager.Allocate(MemoryTag::CSV_READER, MaxValue<idx_t>(Storage::BLOCK_SIZE, buffer_size),
 	                                 can_destroy, &block);
 }
@@ -61,20 +63,21 @@ idx_t CSVBuffer::GetBufferSize() {
 
 void CSVBuffer::Reload(CSVFileHandle &file_handle) {
 	AllocateBuffer(actual_buffer_size);
+	// If we can seek, we seek and return the correct pointers
 	file_handle.Seek(global_csv_start);
 	file_handle.Read(handle.Ptr(), actual_buffer_size);
 }
 
 shared_ptr<CSVBufferHandle> CSVBuffer::Pin(CSVFileHandle &file_handle, bool &has_seeked) {
 	auto &buffer_manager = BufferManager::GetBufferManager(context);
-	if (can_seek && block->IsUnloaded()) {
+	if (!is_pipe && block->IsUnloaded()) {
 		// We have to reload it from disk
 		block = nullptr;
 		Reload(file_handle);
 		has_seeked = true;
 	}
-	return make_shared<CSVBufferHandle>(buffer_manager.Pin(block), actual_buffer_size, last_buffer, file_number,
-	                                    buffer_idx);
+	return make_shared<CSVBufferHandle>(buffer_manager.Pin(block), actual_buffer_size, requested_size, last_buffer,
+	                                    file_number, buffer_idx);
 }
 
 void CSVBuffer::Unpin() {

package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp

@@ -8,6 +8,7 @@ CSVBufferManager::CSVBufferManager(ClientContext &context_p, const CSVReaderOpti
     : context(context_p), file_idx(file_idx_p), file_path(file_path_p), buffer_size(CSVBuffer::CSV_BUFFER_SIZE) {
 	D_ASSERT(!file_path.empty());
 	file_handle = ReadCSV::OpenCSV(file_path, options.compression, context);
+	is_pipe = file_handle->IsPipe();
 	skip_rows = options.dialect_options.skip_rows.GetValue();
 	auto file_size = file_handle->FileSize();
 	if (file_size > 0 && file_size < buffer_size) {
@@ -63,6 +64,15 @@ bool CSVBufferManager::ReadNextAndCacheIt() {
 
 shared_ptr<CSVBufferHandle> CSVBufferManager::GetBuffer(const idx_t pos) {
 	lock_guard<mutex> parallel_lock(main_mutex);
+	if (pos == 0 && done && cached_buffers.empty()) {
+		if (is_pipe) {
+			throw InvalidInputException("Recursive CTEs are not allowed when using piped csv files");
+		}
+		// This is a recursive CTE, we have to reset out whole buffer
+		done = false;
+		file_handle->Reset();
+		Initialize();
+	}
 	while (pos >= cached_buffers.size()) {
 		if (done) {
 			return nullptr;
@@ -71,7 +81,9 @@ shared_ptr<CSVBufferHandle> CSVBufferManager::GetBuffer(const idx_t pos) {
 			done = true;
 		}
 	}
-	if (pos != 0) {
+	if (pos != 0 && (sniffing || file_handle->CanSeek())) {
+		// We don't need to unpin the buffers here if we are not sniffing since we
+		// control it per-thread on the scan
 		if (cached_buffers[pos - 1]) {
 			cached_buffers[pos - 1]->Unpin();
 		}
@@ -98,6 +110,12 @@ void CSVBufferManager::ResetBuffer(const idx_t buffer_idx) {
 	}
 	// We only reset if previous one was also already reset
 	if (buffer_idx > 0 && !cached_buffers[buffer_idx - 1]) {
+		if (cached_buffers[buffer_idx]->last_buffer) {
+			// We clear the whole shebang
+			cached_buffers.clear();
+			reset_when_possible.clear();
+			return;
+		}
 		cached_buffers[buffer_idx].reset();
 		idx_t cur_buffer = buffer_idx + 1;
 		while (reset_when_possible.find(cur_buffer) != reset_when_possible.end()) {
@@ -122,6 +140,19 @@ bool CSVBufferManager::Done() {
 	return done;
 }
 
+void CSVBufferManager::ResetBufferManager() {
+	if (!file_handle->IsPipe()) {
+		// If this is not a pipe we reset the buffer manager and restart it when doing the actual scan
+		cached_buffers.clear();
+		reset_when_possible.clear();
+		file_handle->Reset();
+		last_buffer = nullptr;
+		done = false;
+		global_csv_pos = 0;
+		Initialize();
+	}
+}
+
 string CSVBufferManager::GetFilePath() {
 	return file_path;
 }

package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp

@@ -9,12 +9,13 @@ CSVFileHandle::CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptr<Fi
 	can_seek = file_handle->CanSeek();
 	on_disk_file = file_handle->OnDiskFile();
 	file_size = file_handle->GetFileSize();
+	is_pipe = file_handle->IsPipe();
 	uncompressed = compression == FileCompressionType::UNCOMPRESSED;
 }
 
 unique_ptr<FileHandle> CSVFileHandle::OpenFileHandle(FileSystem &fs, Allocator &allocator, const string &path,
                                                      FileCompressionType compression) {
-	auto file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression);
+	auto file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | compression);
 	if (file_handle->CanSeek()) {
 		file_handle->Reset();
 	}
@@ -33,7 +34,10 @@ bool CSVFileHandle::CanSeek() {
 
 void CSVFileHandle::Seek(idx_t position) {
 	if (!can_seek) {
-		throw InternalException("Cannot seek in this file");
+		if (is_pipe) {
+			throw InternalException("Trying to seek a piped CSV File.");
+		}
+		throw InternalException("Trying to seek a compressed CSV File.");
 	}
 	file_handle->Seek(position);
 }
@@ -42,6 +46,16 @@ bool CSVFileHandle::OnDiskFile() {
 	return on_disk_file;
 }
 
+void CSVFileHandle::Reset() {
+	file_handle->Reset();
+	finished = false;
+	requested_bytes = 0;
+}
+
+bool CSVFileHandle::IsPipe() {
+	return is_pipe;
+}
+
 idx_t CSVFileHandle::FileSize() {
 	return file_size;
 }

package/src/duckdb/src/execution/operator/csv_scanner/scanner/scanner_boundary.cpp

@@ -31,19 +31,19 @@ CSVIterator::CSVIterator() : is_set(false) {
 
 void CSVBoundary::Print() {
 #ifndef DUCKDB_DISABLE_PRINT
-	std::cout << "---Boundary: " << boundary_idx << " ---" << std::endl;
-	std::cout << "File Index:: " << file_idx << std::endl;
-	std::cout << "Buffer Index: " << buffer_idx << std::endl;
-	std::cout << "Buffer Pos: " << buffer_pos << std::endl;
-	std::cout << "End Pos: " << end_pos << std::endl;
-	std::cout << "------------" << end_pos << std::endl;
+	std::cout << "---Boundary: " << boundary_idx << " ---" << '\n';
+	std::cout << "File Index:: " << file_idx << '\n';
+	std::cout << "Buffer Index: " << buffer_idx << '\n';
+	std::cout << "Buffer Pos: " << buffer_pos << '\n';
+	std::cout << "End Pos: " << end_pos << '\n';
+	std::cout << "------------" << end_pos << '\n';
 #endif
 }
 
 void CSVIterator::Print() {
 #ifndef DUCKDB_DISABLE_PRINT
 	boundary.Print();
-	std::cout << "Is set: " << is_set << std::endl;
+	std::cout << "Is set: " << is_set << '\n';
 #endif
 }