duckdb 0.8.2-dev3458.0 → 0.8.2-dev3949.0

package/src/duckdb/src/execution/operator/persistent/csv_buffer.cpp

@@ -1,72 +0,0 @@
-#include "duckdb/execution/operator/persistent/csv_buffer.hpp"
-#include "duckdb/common/string_util.hpp"
-
-namespace duckdb {
-
-CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
-                     idx_t &global_csv_current_position, idx_t file_number_p)
-    : context(context), first_buffer(true), file_number(file_number_p) {
-	this->handle = AllocateBuffer(buffer_size_p);
-
-	auto buffer = Ptr();
-	actual_size = file_handle.Read(buffer, buffer_size_p);
-	global_csv_start = global_csv_current_position;
-	global_csv_current_position += actual_size;
-	if (actual_size >= 3 && buffer[0] == '\xEF' && buffer[1] == '\xBB' && buffer[2] == '\xBF') {
-		start_position += 3;
-	}
-	last_buffer = file_handle.FinishedReading();
-}
-
-CSVBuffer::CSVBuffer(ClientContext &context, BufferHandle buffer_p, idx_t buffer_size_p, idx_t actual_size_p,
-                     bool final_buffer, idx_t global_csv_current_position, idx_t file_number_p)
-    : context(context), handle(std::move(buffer_p)), actual_size(actual_size_p), last_buffer(final_buffer),
-      global_csv_start(global_csv_current_position), file_number(file_number_p) {
-}
-
-unique_ptr<CSVBuffer> CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t &global_csv_current_position,
-                                      idx_t file_number_p) {
-	auto next_buffer = AllocateBuffer(buffer_size);
-	idx_t next_buffer_actual_size = file_handle.Read(next_buffer.Ptr(), buffer_size);
-	if (next_buffer_actual_size == 0) {
-		// We are done reading
-		return nullptr;
-	}
-
-	auto next_csv_buffer =
-	    make_uniq<CSVBuffer>(context, std::move(next_buffer), buffer_size, next_buffer_actual_size,
-	                         file_handle.FinishedReading(), global_csv_current_position, file_number_p);
-	global_csv_current_position += next_buffer_actual_size;
-	return next_csv_buffer;
-}
-
-BufferHandle CSVBuffer::AllocateBuffer(idx_t buffer_size) {
-	auto &buffer_manager = BufferManager::GetBufferManager(context);
-	return buffer_manager.Allocate(MaxValue<idx_t>(Storage::BLOCK_SIZE, buffer_size));
-}
-
-idx_t CSVBuffer::GetBufferSize() {
-	return actual_size;
-}
-
-idx_t CSVBuffer::GetStart() {
-	return start_position;
-}
-
-bool CSVBuffer::IsCSVFileLastBuffer() {
-	return last_buffer;
-}
-
-bool CSVBuffer::IsCSVFileFirstBuffer() {
-	return first_buffer;
-}
-
-idx_t CSVBuffer::GetCSVGlobalStart() {
-	return global_csv_start;
-}
-
-idx_t CSVBuffer::GetFileNumber() {
-	return file_number;
-}
-
-} // namespace duckdb

package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp

@@ -1,158 +0,0 @@
-#include "duckdb/execution/operator/persistent/csv_file_handle.hpp"
-
-namespace duckdb {
-
-CSVFileHandle::CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptr<FileHandle> file_handle_p,
-                             const string &path_p, FileCompressionType compression, bool enable_reset)
-    : fs(fs), allocator(allocator), file_handle(std::move(file_handle_p)), path(path_p), compression(compression),
-      reset_enabled(enable_reset) {
-	can_seek = file_handle->CanSeek();
-	on_disk_file = file_handle->OnDiskFile();
-	file_size = file_handle->GetFileSize();
-}
-
-unique_ptr<FileHandle> CSVFileHandle::OpenFileHandle(FileSystem &fs, Allocator &allocator, const string &path,
-                                                     FileCompressionType compression) {
-	auto file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression);
-	if (file_handle->CanSeek()) {
-		file_handle->Reset();
-	}
-	return file_handle;
-}
-
-unique_ptr<CSVFileHandle> CSVFileHandle::OpenFile(FileSystem &fs, Allocator &allocator, const string &path,
-                                                  FileCompressionType compression, bool enable_reset) {
-	auto file_handle = CSVFileHandle::OpenFileHandle(fs, allocator, path, compression);
-	return make_uniq<CSVFileHandle>(fs, allocator, std::move(file_handle), path, compression, enable_reset);
-}
-
-bool CSVFileHandle::CanSeek() {
-	return can_seek;
-}
-
-void CSVFileHandle::Seek(idx_t position) {
-	if (!can_seek) {
-		throw InternalException("Cannot seek in this file");
-	}
-	file_handle->Seek(position);
-}
-
-idx_t CSVFileHandle::SeekPosition() {
-	if (!can_seek) {
-		throw InternalException("Cannot seek in this file");
-	}
-	return file_handle->SeekPosition();
-}
-
-void CSVFileHandle::Reset() {
-	requested_bytes = 0;
-	read_position = 0;
-	if (can_seek) {
-		// we can seek - reset the file handle
-		file_handle->Reset();
-	} else if (on_disk_file) {
-		// we cannot seek but it is an on-disk file - re-open the file
-		file_handle = CSVFileHandle::OpenFileHandle(fs, allocator, path, compression);
-	} else {
-		if (!reset_enabled) {
-			throw InternalException("Reset called but reset is not enabled for this CSV Handle");
-		}
-		read_position = 0;
-	}
-}
-bool CSVFileHandle::OnDiskFile() {
-	return on_disk_file;
-}
-
-idx_t CSVFileHandle::FileSize() {
-	return file_size;
-}
-
-bool CSVFileHandle::FinishedReading() {
-	return requested_bytes >= file_size;
-}
-
-idx_t CSVFileHandle::Read(void *buffer, idx_t nr_bytes) {
-	requested_bytes += nr_bytes;
-	if (on_disk_file || can_seek) {
-		// if this is a plain file source OR we can seek we are not caching anything
-		return file_handle->Read(buffer, nr_bytes);
-	}
-	// not a plain file source: we need to do some bookkeeping around the reset functionality
-	idx_t result_offset = 0;
-	if (read_position < buffer_size) {
-		// we need to read from our cached buffer
-		auto buffer_read_count = MinValue<idx_t>(nr_bytes, buffer_size - read_position);
-		memcpy(buffer, cached_buffer.get() + read_position, buffer_read_count);
-		result_offset += buffer_read_count;
-		read_position += buffer_read_count;
-		if (result_offset == nr_bytes) {
-			return nr_bytes;
-		}
-	} else if (!reset_enabled && cached_buffer.IsSet()) {
-		// reset is disabled, but we still have cached data
-		// we can remove any cached data
-		cached_buffer.Reset();
-		buffer_size = 0;
-		buffer_capacity = 0;
-		read_position = 0;
-	}
-	// we have data left to read from the file
-	// read directly into the buffer
-	auto bytes_read = file_handle->Read(char_ptr_cast(buffer) + result_offset, nr_bytes - result_offset);
-	file_size = file_handle->GetFileSize();
-	read_position += bytes_read;
-	if (reset_enabled) {
-		// if reset caching is enabled, we need to cache the bytes that we have read
-		if (buffer_size + bytes_read >= buffer_capacity) {
-			// no space; first enlarge the buffer
-			buffer_capacity = MaxValue<idx_t>(NextPowerOfTwo(buffer_size + bytes_read), buffer_capacity * 2);
-
-			auto new_buffer = allocator.Allocate(buffer_capacity);
-			if (buffer_size > 0) {
-				memcpy(new_buffer.get(), cached_buffer.get(), buffer_size);
-			}
-			cached_buffer = std::move(new_buffer);
-		}
-		memcpy(cached_buffer.get() + buffer_size, char_ptr_cast(buffer) + result_offset, bytes_read);
-		buffer_size += bytes_read;
-	}
-
-	return result_offset + bytes_read;
-}
-
-string CSVFileHandle::ReadLine() {
-	bool carriage_return = false;
-	string result;
-	char buffer[1];
-	while (true) {
-		idx_t bytes_read = Read(buffer, 1);
-		if (bytes_read == 0) {
-			return result;
-		}
-		if (carriage_return) {
-			if (buffer[0] != '\n') {
-				if (!file_handle->CanSeek()) {
-					throw BinderException(
-					    "Carriage return newlines not supported when reading CSV files in which we cannot seek");
-				}
-				file_handle->Seek(file_handle->SeekPosition() - 1);
-				return result;
-			}
-		}
-		if (buffer[0] == '\n') {
-			return result;
-		}
-		if (buffer[0] != '\r') {
-			result += buffer[0];
-		} else {
-			carriage_return = true;
-		}
-	}
-}
-
-void CSVFileHandle::DisableReset() {
-	this->reset_enabled = false;
-}
-
-} // namespace duckdb

package/src/duckdb/src/execution/partitionable_hashtable.cpp

@@ -1,207 +0,0 @@
-#include "duckdb/execution/partitionable_hashtable.hpp"
-
-#include "duckdb/common/radix_partitioning.hpp"
-
-namespace duckdb {
-
-RadixPartitionInfo::RadixPartitionInfo(const idx_t n_partitions_upper_bound)
-    : n_partitions(PreviousPowerOfTwo(n_partitions_upper_bound)),
-      radix_bits(RadixPartitioning::RadixBits(n_partitions)), radix_mask(RadixPartitioning::Mask(radix_bits)),
-      radix_shift(RadixPartitioning::Shift(radix_bits)) {
-
-	D_ASSERT(radix_bits <= RadixPartitioning::MAX_RADIX_BITS);
-	D_ASSERT(n_partitions > 0);
-	D_ASSERT(n_partitions == RadixPartitioning::NumberOfPartitions(radix_bits));
-	D_ASSERT(IsPowerOfTwo(n_partitions));
-}
-
-PartitionableHashTable::PartitionableHashTable(ClientContext &context, Allocator &allocator,
-                                               RadixPartitionInfo &partition_info_p, vector<LogicalType> group_types_p,
-                                               vector<LogicalType> payload_types_p,
-                                               vector<BoundAggregateExpression *> bindings_p)
-    : context(context), allocator(allocator), group_types(std::move(group_types_p)),
-      payload_types(std::move(payload_types_p)), bindings(std::move(bindings_p)), is_partitioned(false),
-      partition_info(partition_info_p), hashes(LogicalType::HASH), hashes_subset(LogicalType::HASH) {
-
-	sel_vectors.resize(partition_info.n_partitions);
-	sel_vector_sizes.resize(partition_info.n_partitions);
-	group_subset.Initialize(allocator, group_types);
-	if (!payload_types.empty()) {
-		payload_subset.Initialize(allocator, payload_types);
-	}
-
-	for (hash_t r = 0; r < partition_info.n_partitions; r++) {
-		sel_vectors[r].Initialize();
-	}
-
-	RowLayout layout;
-	layout.Initialize(group_types, AggregateObject::CreateAggregateObjects(bindings));
-	tuple_size = layout.GetRowWidth();
-}
-
-HtEntryType PartitionableHashTable::GetHTEntrySize() {
-	// we need at least STANDARD_VECTOR_SIZE entries to fit in the hash table
-	if (GroupedAggregateHashTable::GetMaxCapacity(HtEntryType::HT_WIDTH_32, tuple_size) < STANDARD_VECTOR_SIZE) {
-		return HtEntryType::HT_WIDTH_64;
-	}
-	return HtEntryType::HT_WIDTH_32;
-}
-
-bool OverMemoryLimit(ClientContext &context, const bool is_partitioned, const RadixPartitionInfo &partition_info,
-                     const GroupedAggregateHashTable &ht) {
-	const auto n_partitions = is_partitioned ? partition_info.n_partitions : 1;
-	const auto max_memory = BufferManager::GetBufferManager(context).GetMaxMemory();
-	const auto num_threads = TaskScheduler::GetScheduler(context).NumberOfThreads();
-	const auto memory_per_partition = 0.6 * max_memory / num_threads / n_partitions;
-	return ht.TotalSize() > memory_per_partition;
-}
-
-idx_t PartitionableHashTable::ListAddChunk(HashTableList &list, DataChunk &groups, Vector &group_hashes,
-                                           DataChunk &payload, const unsafe_vector<idx_t> &filter) {
-	// If this is false, a single AddChunk would overflow the max capacity
-	D_ASSERT(list.empty() || groups.size() <= list.back()->MaxCapacity());
-	if (list.empty() || list.back()->Count() + groups.size() >= list.back()->MaxCapacity() ||
-	    OverMemoryLimit(context, is_partitioned, partition_info, *list.back())) {
-		idx_t new_capacity = GroupedAggregateHashTable::InitialCapacity();
-		if (!list.empty()) {
-			new_capacity = list.back()->Capacity();
-			// early release first part of ht and prevent adding of more data
-			list.back()->Finalize();
-		}
-		list.push_back(make_uniq<GroupedAggregateHashTable>(context, allocator, group_types, payload_types, bindings,
-		                                                    GetHTEntrySize(), new_capacity));
-	}
-	return list.back()->AddChunk(append_state, groups, group_hashes, payload, filter);
-}
-
-idx_t PartitionableHashTable::AddChunk(DataChunk &groups, DataChunk &payload, bool do_partition,
-                                       const unsafe_vector<idx_t> &filter) {
-	groups.Hash(hashes);
-
-	// we partition when we are asked to or when the unpartitioned ht runs out of space
-	if (!IsPartitioned() && do_partition) {
-		Partition(false);
-	}
-
-	if (!IsPartitioned()) {
-		return ListAddChunk(unpartitioned_hts, groups, hashes, payload, filter);
-	}
-
-	// makes no sense to do this with 1 partition
-	D_ASSERT(partition_info.n_partitions > 0);
-
-	for (hash_t r = 0; r < partition_info.n_partitions; r++) {
-		sel_vector_sizes[r] = 0;
-	}
-
-	hashes.Flatten(groups.size());
-	auto hashes_ptr = FlatVector::GetData<hash_t>(hashes);
-
-	// Determine for every partition how much data will be sinked into it
-	for (idx_t i = 0; i < groups.size(); i++) {
-		auto partition = partition_info.GetHashPartition(hashes_ptr[i]);
-		D_ASSERT(partition < partition_info.n_partitions);
-		sel_vectors[partition].set_index(sel_vector_sizes[partition]++, i);
-	}
-
-#ifdef DEBUG
-	// make sure we have lost no rows
-	idx_t total_count = 0;
-	for (idx_t r = 0; r < partition_info.n_partitions; r++) {
-		total_count += sel_vector_sizes[r];
-	}
-	D_ASSERT(total_count == groups.size());
-#endif
-	idx_t group_count = 0;
-	for (hash_t r = 0; r < partition_info.n_partitions; r++) {
-		group_subset.Slice(groups, sel_vectors[r], sel_vector_sizes[r]);
-		if (!payload_types.empty()) {
-			payload_subset.Slice(payload, sel_vectors[r], sel_vector_sizes[r]);
-		} else {
-			payload_subset.SetCardinality(sel_vector_sizes[r]);
-		}
-		hashes_subset.Slice(hashes, sel_vectors[r], sel_vector_sizes[r]);
-
-		group_count += ListAddChunk(radix_partitioned_hts[r], group_subset, hashes_subset, payload_subset, filter);
-	}
-	return group_count;
-}
-
-void PartitionableHashTable::Partition(bool sink_done) {
-	D_ASSERT(!IsPartitioned());
-	D_ASSERT(radix_partitioned_hts.empty());
-	D_ASSERT(partition_info.n_partitions > 1);
-
-	vector<GroupedAggregateHashTable *> partition_hts(partition_info.n_partitions);
-	radix_partitioned_hts.resize(partition_info.n_partitions);
-	for (auto &unpartitioned_ht : unpartitioned_hts) {
-		for (idx_t r = 0; r < partition_info.n_partitions; r++) {
-			radix_partitioned_hts[r].push_back(make_uniq<GroupedAggregateHashTable>(
-			    context, allocator, group_types, payload_types, bindings, GetHTEntrySize()));
-			partition_hts[r] = radix_partitioned_hts[r].back().get();
-		}
-		unpartitioned_ht->Partition(partition_hts, partition_info.radix_bits, sink_done);
-		unpartitioned_ht.reset();
-	}
-	unpartitioned_hts.clear();
-	is_partitioned = true;
-}
-
-bool PartitionableHashTable::IsPartitioned() {
-	return is_partitioned;
-}
-
-HashTableList PartitionableHashTable::GetPartition(idx_t partition) {
-	D_ASSERT(IsPartitioned());
-	D_ASSERT(partition < partition_info.n_partitions);
-	D_ASSERT(radix_partitioned_hts.size() > partition);
-	return std::move(radix_partitioned_hts[partition]);
-}
-
-HashTableList PartitionableHashTable::GetUnpartitioned() {
-	D_ASSERT(!IsPartitioned());
-	return std::move(unpartitioned_hts);
-}
-
-idx_t PartitionableHashTable::GetPartitionCount(idx_t partition) const {
-	idx_t total_size = 0;
-	for (const auto &ht : radix_partitioned_hts[partition]) {
-		total_size += ht->Count();
-	}
-	return total_size;
-}
-
-idx_t PartitionableHashTable::GetPartitionSize(idx_t partition) const {
-	idx_t total_size = 0;
-	for (const auto &ht : radix_partitioned_hts[partition]) {
-		total_size += ht->DataSize();
-	}
-	return total_size;
-}
-
-void PartitionableHashTable::Finalize() {
-	if (IsPartitioned()) {
-		for (auto &ht_list : radix_partitioned_hts) {
-			for (auto &ht : ht_list) {
-				D_ASSERT(ht);
-				ht->Finalize();
-			}
-		}
-	} else {
-		for (auto &ht : unpartitioned_hts) {
-			D_ASSERT(ht);
-			ht->Finalize();
-		}
-	}
-}
-
-void PartitionableHashTable::Append(GroupedAggregateHashTable &ht) {
-	if (unpartitioned_hts.empty()) {
-		unpartitioned_hts.push_back(make_uniq<GroupedAggregateHashTable>(context, allocator, group_types, payload_types,
-		                                                                 bindings, GetHTEntrySize(),
-		                                                                 GroupedAggregateHashTable::InitialCapacity()));
-	}
-	unpartitioned_hts.back()->Append(ht);
-}
-
-} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/persistent/buffered_csv_reader.hpp

@@ -1,133 +0,0 @@
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/execution/operator/persistent/base_csv_reader.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-#pragma once
-
-#include "duckdb/execution/operator/persistent/base_csv_reader.hpp"
-
-namespace duckdb {
-struct CopyInfo;
-struct CSVFileHandle;
-struct FileHandle;
-struct StrpTimeFormat;
-
-class FileOpener;
-class FileSystem;
-
-//! The shifts array allows for linear searching of multi-byte values. For each position, it determines the next
-//! position given that we encounter a byte with the given value.
-/*! For example, if we have a string "ABAC", the shifts array will have the following values:
- *  [0] --> ['A'] = 1, all others = 0
- *  [1] --> ['B'] = 2, ['A'] = 1, all others = 0
- *  [2] --> ['A'] = 3, all others = 0
- *  [3] --> ['C'] = 4 (match), 'B' = 2, 'A' = 1, all others = 0
- * Suppose we then search in the following string "ABABAC", our progression will be as follows:
- * 'A' -> [1], 'B' -> [2], 'A' -> [3], 'B' -> [2], 'A' -> [3], 'C' -> [4] (match!)
- */
-struct TextSearchShiftArray {
-	TextSearchShiftArray();
-	explicit TextSearchShiftArray(string search_term);
-
-	inline bool Match(uint8_t &position, uint8_t byte_value) {
-		if (position >= length) {
-			return false;
-		}
-		position = shifts[position * 255 + byte_value];
-		return position == length;
-	}
-
-	idx_t length;
-	unique_ptr<uint8_t[]> shifts;
-};
-
-//! Buffered CSV reader is a class that reads values from a stream and parses them as a CSV file
-class BufferedCSVReader : public BaseCSVReader {
-	//! Initial buffer read size; can be extended for long lines
-	static constexpr idx_t INITIAL_BUFFER_SIZE = 16384;
-	//! Larger buffer size for non disk files
-	static constexpr idx_t INITIAL_BUFFER_SIZE_LARGE = 10000000; // 10MB
-
-public:
-	BufferedCSVReader(ClientContext &context, BufferedCSVReaderOptions options,
-	                  const vector<LogicalType> &requested_types = vector<LogicalType>());
-	BufferedCSVReader(ClientContext &context, string filename, BufferedCSVReaderOptions options,
-	                  const vector<LogicalType> &requested_types = vector<LogicalType>());
-	virtual ~BufferedCSVReader() {
-	}
-
-	unsafe_unique_array<char> buffer;
-	idx_t buffer_size;
-	idx_t position;
-	idx_t start = 0;
-
-	vector<unsafe_unique_array<char>> cached_buffers;
-
-	unique_ptr<CSVFileHandle> file_handle;
-
-	TextSearchShiftArray delimiter_search, escape_search, quote_search;
-
-public:
-	//! Extract a single DataChunk from the CSV file and stores it in insert_chunk
-	void ParseCSV(DataChunk &insert_chunk);
-	static string ColumnTypesError(case_insensitive_map_t<idx_t> sql_types_per_column, const vector<string> &names);
-
-private:
-	//! Initialize Parser
-	void Initialize(const vector<LogicalType> &requested_types);
-	//! Skips skip_rows, reads header row from input stream
-	void SkipRowsAndReadHeader(idx_t skip_rows, bool skip_header);
-	//! Jumps back to the beginning of input stream and resets necessary internal states
-	void JumpToBeginning(idx_t skip_rows, bool skip_header);
-	//! Resets the buffer
-	void ResetBuffer();
-	//! Resets the steam
-	void ResetStream();
-	//! Reads a new buffer from the CSV file if the current one has been exhausted
-	bool ReadBuffer(idx_t &start, idx_t &line_start);
-	//! Jumps back to the beginning of input stream and resets necessary internal states
-	bool JumpToNextSample();
-	//! Initializes the TextSearchShiftArrays for complex parser
-	void PrepareComplexParser();
-	//! Try to parse a single datachunk from the file. Throws an exception if anything goes wrong.
-	void ParseCSV(ParserMode mode);
-	//! Try to parse a single datachunk from the file. Returns whether or not the parsing is successful
-	bool TryParseCSV(ParserMode mode);
-	//! Extract a single DataChunk from the CSV file and stores it in insert_chunk
-	bool TryParseCSV(ParserMode mode, DataChunk &insert_chunk, string &error_message);
-
-	//! Parses a CSV file with a one-byte delimiter, escape and quote character
-	bool TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message);
-	//! Parses more complex CSV files with multi-byte delimiters, escapes or quotes
-	bool TryParseComplexCSV(DataChunk &insert_chunk, string &error_message);
-	//! Sniffs CSV dialect and determines skip rows, header row, column types and column names
-	vector<LogicalType> SniffCSV(const vector<LogicalType> &requested_types);
-
-	//! First phase of auto detection: detect CSV dialect (i.e. delimiter, quote rules, etc)
-	void DetectDialect(const vector<LogicalType> &requested_types, BufferedCSVReaderOptions &original_options,
-	                   vector<BufferedCSVReaderOptions> &info_candidates, idx_t &best_num_cols);
-	//! Second phase of auto detection: detect candidate types for each column
-	void DetectCandidateTypes(const vector<LogicalType> &type_candidates,
-	                          const map<LogicalTypeId, vector<const char *>> &format_template_candidates,
-	                          const vector<BufferedCSVReaderOptions> &info_candidates,
-	                          BufferedCSVReaderOptions &original_options, idx_t best_num_cols,
-	                          vector<vector<LogicalType>> &best_sql_types_candidates,
-	                          std::map<LogicalTypeId, vector<string>> &best_format_candidates,
-	                          DataChunk &best_header_row);
-	//! Third phase of auto detection: detect header of CSV file
-	void DetectHeader(const vector<vector<LogicalType>> &best_sql_types_candidates, const DataChunk &best_header_row);
-	//! Fourth phase of auto detection: refine the types of each column and select which types to use for each column
-	vector<LogicalType> RefineTypeDetection(const vector<LogicalType> &type_candidates,
-	                                        const vector<LogicalType> &requested_types,
-	                                        vector<vector<LogicalType>> &best_sql_types_candidates,
-	                                        map<LogicalTypeId, vector<string>> &best_format_candidates);
-
-	//! Skip Empty lines for tables with over one column
-	void SkipEmptyLines();
-};
-
-} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_buffer.hpp

@@ -1,74 +0,0 @@
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/execution/operator/persistent/csv_buffer.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-#pragma once
-
-#include "duckdb/common/constants.hpp"
-#include "duckdb/execution/operator/persistent/csv_file_handle.hpp"
-#include "duckdb/storage/buffer_manager.hpp"
-
-namespace duckdb {
-
-class CSVBuffer {
-public:
-	//! Colossal buffer size for multi-threading
-	static constexpr idx_t INITIAL_BUFFER_SIZE_COLOSSAL = 32000000; // 32MB
-
-	//! Constructor for Initial Buffer
-	CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
-	          idx_t &global_csv_current_position, idx_t file_number);
-
-	//! Constructor for `Next()` Buffers
-	CSVBuffer(ClientContext &context, BufferHandle handle, idx_t buffer_size_p, idx_t actual_size_p, bool final_buffer,
-	          idx_t global_csv_current_position, idx_t file_number);
-
-	//! Creates a new buffer with the next part of the CSV File
-	unique_ptr<CSVBuffer> Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t &global_csv_current_position,
-	                           idx_t file_number);
-
-	//! Gets the buffer actual size
-	idx_t GetBufferSize();
-
-	//! Gets the start position of the buffer, only relevant for the first time it's scanned
-	idx_t GetStart();
-
-	//! If this buffer is the last buffer of the CSV File
-	bool IsCSVFileLastBuffer();
-
-	//! If this buffer is the first buffer of the CSV File
-	bool IsCSVFileFirstBuffer();
-
-	idx_t GetCSVGlobalStart();
-
-	idx_t GetFileNumber();
-
-	BufferHandle AllocateBuffer(idx_t buffer_size);
-
-	char *Ptr() {
-		return char_ptr_cast(handle.Ptr());
-	}
-
-private:
-	ClientContext &context;
-
-	BufferHandle handle;
-	//! Actual size can be smaller than the buffer size in case we allocate it too optimistically.
-	idx_t actual_size;
-	//! We need to check for Byte Order Mark, to define the start position of this buffer
-	//! https://en.wikipedia.org/wiki/Byte_order_mark#UTF-8
-	idx_t start_position = 0;
-	//! If this is the last buffer of the CSV File
-	bool last_buffer = false;
-	//! If this is the first buffer of the CSV File
-	bool first_buffer = false;
-	//! Global position from the CSV File where this buffer starts
-	idx_t global_csv_start = 0;
-	//! Number of the file that is in this buffer
-	idx_t file_number = 0;
-};
-} // namespace duckdb