duckdb 0.10.1-dev9.0 → 0.10.2-dev0.0

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/quote_rules.hpp

@@ -0,0 +1,21 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/quote_rules.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/vector.hpp"
+
+namespace duckdb {
+//! Different Rules regarding possible combinations of Quote and Escape Values for CSV Dialects.
+//! Each rule has a comment on the possible combinations.
+enum class QuoteRule : uint8_t {
+	QUOTES_RFC = 0,   //! quote = " escape = (\0 || " || ')
+	QUOTES_OTHER = 1, //! quote = ( " || ' ) escape = '\\'
+	NO_QUOTES = 2     //! quote = \0 escape = \0
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/scanner_boundary.hpp

@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/scanner_boundary.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_file_handle.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_state_machine.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/common/multi_file_reader.hpp"
+#include "duckdb/common/fast_mem.hpp"
+
+//! We all need boundaries every now and then, CSV Scans also need them
+//! This class keeps track of what a scan should read, so which buffer and from where to where
+//! As in real life, much like my childhood country, the rules are not really enforced.
+//! So the end boundaries of a Scanner Boundary can and will be pushed.
+//! In practice this means that a scanner is tolerated to read one line over it's end.
+namespace duckdb {
+
+//! Information stored in the buffer
+struct CSVBoundary {
+	CSVBoundary(idx_t file_idx, idx_t buffer_idx, idx_t buffer_pos, idx_t boundary_idx, idx_t end_pos);
+	CSVBoundary();
+	void Print();
+	//! File index where we start scanning [0-idx], a scanner can never go over one file.
+	idx_t file_idx = 0;
+	//! Start Buffer index of the file where we start scanning
+	idx_t buffer_idx = 0;
+	//! Start Buffer position of the buffer of the file where we start scanning
+	//! This position moves as we move through the buffer
+	idx_t buffer_pos = 0;
+	//! The boundary index relative to the total scan, only used for parallel reading to enforce
+	//! Insertion Order
+	idx_t boundary_idx = 0;
+	//! Last position this iterator should read.
+	idx_t end_pos;
+};
+
+struct CSVPosition {
+	CSVPosition(idx_t file_idx, idx_t buffer_idx, idx_t buffer_pos);
+	CSVPosition();
+	//! File index where we start scanning [0-idx], a scanner can never go over one file.
+	idx_t file_idx = 0;
+	//! Start Buffer index of the file where we start scanning
+	idx_t buffer_idx = 0;
+	//! Start Buffer position of the buffer of the file where we start scanning
+	//! This position moves as we move through the buffer
+	idx_t buffer_pos = 0;
+};
+struct CSVIterator {
+public:
+	CSVIterator(idx_t file_idx, idx_t buffer_idx, idx_t buffer_pos, idx_t boundary_idx, idx_t buffer_size);
+
+	CSVIterator();
+
+	void Print();
+	//! Moves the boundary to the next one to be scanned, if there are no next boundaries, it returns False
+	//! Otherwise, if there are boundaries, it returns True
+	bool Next(CSVBufferManager &buffer_manager);
+	//! If boundary is set
+	bool IsBoundarySet() const;
+
+	//! Getters
+	idx_t GetEndPos() const;
+	idx_t GetFileIdx() const;
+	idx_t GetBufferIdx() const;
+	idx_t GetBoundaryIdx() const;
+
+	void SetCurrentPositionToBoundary();
+
+	void SetStart(idx_t pos);
+
+	//! 8 MB TODO: Should benchmarks other values
+	static constexpr idx_t BYTES_PER_THREAD = 8000000;
+
+	CSVPosition pos;
+
+	bool done = false;
+
+private:
+	//! The original setting
+	CSVBoundary boundary;
+	//! Sometimes life knows no boundaries.
+	//! The boundaries don't have to be set for single-threaded execution.
+	bool is_set;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/skip_scanner.hpp

@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/skip_scanner.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_state_machine.hpp"
+#include "duckdb/execution/operator/csv_scanner/scanner_boundary.hpp"
+#include "duckdb/execution/operator/csv_scanner/string_value_scanner.hpp"
+#include "duckdb/execution/operator/csv_scanner/base_scanner.hpp"
+
+namespace duckdb {
+
+class SkipResult : public ScannerResult {
+public:
+	SkipResult(CSVStates &states, CSVStateMachine &state_machine, idx_t rows_to_skip);
+
+	idx_t row_count = 0;
+	idx_t rows_to_skip;
+
+	//! Adds a Value to the result
+	static inline void AddValue(SkipResult &result, const idx_t buffer_pos);
+	//! Adds a Row to the result
+	static inline bool AddRow(SkipResult &result, const idx_t buffer_pos);
+	//! Behavior when hitting an invalid state
+	static inline void InvalidState(SkipResult &result);
+	//! Handles EmptyLine states
+	static inline bool EmptyLine(SkipResult &result, const idx_t buffer_pos);
+	//! Handles QuotedNewline State
+	static inline void QuotedNewLine(SkipResult &result);
+	inline void InternalAddRow();
+};
+
+//! Scanner used to skip lines in a CSV File
+class SkipScanner : public BaseScanner {
+public:
+	SkipScanner(shared_ptr<CSVBufferManager> buffer_manager, const shared_ptr<CSVStateMachine> &state_machine,
+	            shared_ptr<CSVErrorHandler> error_handler, idx_t rows_to_skip);
+
+	~SkipScanner() {
+	}
+
+	SkipResult &ParseChunk() override;
+
+	SkipResult &GetResult() override;
+
+private:
+	void Initialize() override;
+
+	void FinalizeChunkProcess() override;
+
+	SkipResult result;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/state_machine_options.hpp

@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/state_machine_options.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/csv_scanner/csv_option.hpp"
+
+namespace duckdb {
+//! Struct that holds the configuration of a CSV State Machine
+//! Basically which char, quote and escape were used to generate it.
+struct CSVStateMachineOptions {
+	CSVStateMachineOptions() {};
+	CSVStateMachineOptions(char delimiter_p, char quote_p, char escape_p, NewLineIdentifier new_line_p)
+	    : delimiter(delimiter_p), quote(quote_p), escape(escape_p), new_line(new_line_p) {};
+
+	//! Delimiter to separate columns within each line
+	CSVOption<char> delimiter = ',';
+	//! Quote used for columns that contain reserved characters, e.g '
+	CSVOption<char> quote = '\"';
+	//! Escape character to escape quote character
+	CSVOption<char> escape = '\0';
+	//! New Line separator
+	CSVOption<NewLineIdentifier> new_line = NewLineIdentifier::NOT_SET;
+
+	bool operator==(const CSVStateMachineOptions &other) const {
+		return delimiter == other.delimiter && quote == other.quote && escape == other.escape &&
+		       new_line == other.new_line;
+	}
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp

@@ -0,0 +1,200 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/string_value_scanner.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_state_machine.hpp"
+#include "duckdb/execution/operator/csv_scanner/scanner_boundary.hpp"
+#include "duckdb/execution/operator/csv_scanner/base_scanner.hpp"
+
+namespace duckdb {
+
+struct CSVBufferUsage {
+	CSVBufferUsage(CSVBufferManager &buffer_manager_p, idx_t buffer_idx_p)
+	    : buffer_manager(buffer_manager_p), buffer_idx(buffer_idx_p) {
+
+	                                        };
+	~CSVBufferUsage() {
+		buffer_manager.ResetBuffer(buffer_idx);
+	}
+	CSVBufferManager &buffer_manager;
+	idx_t buffer_idx;
+};
+
+//! Class that keeps track of line starts, used for line size verification
+class LinePosition {
+public:
+	LinePosition() {
+	}
+	LinePosition(idx_t buffer_idx_p, idx_t buffer_pos_p, idx_t buffer_size_p)
+	    : buffer_pos(buffer_pos_p), buffer_size(buffer_size_p), buffer_idx(buffer_idx_p) {
+	}
+
+	idx_t operator-(const LinePosition &other) {
+		if (other.buffer_idx == buffer_idx) {
+			return buffer_pos - other.buffer_pos;
+		}
+		return other.buffer_size - other.buffer_pos + buffer_pos;
+	}
+	idx_t buffer_pos = 0;
+	idx_t buffer_size = 0;
+	idx_t buffer_idx = 0;
+};
+
+class StringValueResult : public ScannerResult {
+public:
+	StringValueResult(CSVStates &states, CSVStateMachine &state_machine,
+	                  const shared_ptr<CSVBufferHandle> &buffer_handle, Allocator &buffer_allocator, idx_t result_size,
+	                  idx_t buffer_position, CSVErrorHandler &error_hander, CSVIterator &iterator, bool store_line_size,
+	                  shared_ptr<CSVFileScan> csv_file_scan, idx_t &lines_read, bool sniffing);
+
+	~StringValueResult();
+
+	//! Information on the vector
+	unsafe_vector<void *> vector_ptr;
+	unsafe_vector<ValidityMask *> validity_mask;
+
+	//! Variables to iterate over the CSV buffers
+	idx_t last_position;
+	char *buffer_ptr;
+	idx_t buffer_size;
+
+	//! CSV Options that impact the parsing
+	const uint32_t number_of_columns;
+	const bool null_padding;
+	const bool ignore_errors;
+	const char *null_str_ptr;
+	const idx_t null_str_size;
+
+	//! Internal Data Chunk used for flushing
+	DataChunk parse_chunk;
+	idx_t number_of_rows = 0;
+	idx_t cur_col_id = 0;
+	idx_t result_size;
+	//! Information to properly handle errors
+	CSVErrorHandler &error_handler;
+	CSVIterator &iterator;
+	//! Where the previous line started, used to validate the maximum_line_size option
+	LinePosition previous_line_start;
+	LinePosition pre_previous_line_start;
+	bool store_line_size = false;
+	bool added_last_line = false;
+	bool quoted_new_line = false;
+
+	unsafe_unique_array<std::pair<LogicalTypeId, bool>> parse_types;
+	vector<string> names;
+	unordered_map<idx_t, string> cast_errors;
+
+	shared_ptr<CSVFileScan> csv_file_scan;
+	idx_t &lines_read;
+	//! Information regarding projected columns
+	unsafe_unique_array<bool> projected_columns;
+	bool projecting_columns = false;
+	idx_t chunk_col_id = 0;
+
+	//! We must ensure that we keep the buffers alive until processing the query result
+	vector<shared_ptr<CSVBufferHandle>> buffer_handles;
+
+	//! If the current row has an error, we have to skip it
+	bool ignore_current_row = false;
+
+	bool sniffing;
+	//! Specialized code for quoted values, makes sure to remove quotes and escapes
+	static inline void AddQuotedValue(StringValueResult &result, const idx_t buffer_pos);
+	//! Adds a Value to the result
+	static inline void AddValue(StringValueResult &result, const idx_t buffer_pos);
+	//! Adds a Row to the result
+	static inline bool AddRow(StringValueResult &result, const idx_t buffer_pos);
+	//! Behavior when hitting an invalid state
+	static inline void InvalidState(StringValueResult &result);
+	//! Handles QuotedNewline State
+	static inline void QuotedNewLine(StringValueResult &result);
+	void NullPaddingQuotedNewlineCheck();
+	//! Handles EmptyLine states
+	static inline bool EmptyLine(StringValueResult &result, const idx_t buffer_pos);
+	inline bool AddRowInternal();
+
+	void HandleOverLimitRows();
+
+	inline void AddValueToVector(const char *value_ptr, const idx_t size, bool allocate = false);
+
+	Value GetValue(idx_t row_idx, idx_t col_idx);
+
+	DataChunk &ToChunk();
+	//! Resets the state of the result
+	void Reset();
+};
+
+//! Our dialect scanner basically goes over the CSV and actually parses the values to a DuckDB vector of string_t
+class StringValueScanner : public BaseScanner {
+public:
+	StringValueScanner(idx_t scanner_idx, const shared_ptr<CSVBufferManager> &buffer_manager,
+	                   const shared_ptr<CSVStateMachine> &state_machine,
+	                   const shared_ptr<CSVErrorHandler> &error_handler, const shared_ptr<CSVFileScan> &csv_file_scan,
+	                   bool sniffing = false, CSVIterator boundary = {}, idx_t result_size = STANDARD_VECTOR_SIZE);
+
+	StringValueScanner(const shared_ptr<CSVBufferManager> &buffer_manager,
+	                   const shared_ptr<CSVStateMachine> &state_machine,
+	                   const shared_ptr<CSVErrorHandler> &error_handler);
+
+	~StringValueScanner() {
+	}
+
+	StringValueResult &ParseChunk() override;
+
+	//! Flushes the result to the insert_chunk
+	void Flush(DataChunk &insert_chunk);
+
+	//! Function that creates and returns a non-boundary CSV Scanner, can be used for internal csv reading.
+	static unique_ptr<StringValueScanner> GetCSVScanner(ClientContext &context, CSVReaderOptions &options);
+
+	bool FinishedIterator();
+
+	//! Creates a new string with all escaped values removed
+	static string_t RemoveEscape(const char *str_ptr, idx_t end, char escape, Vector &vector);
+
+	//! If we can directly cast the type when consuming the CSV file, or we have to do it later
+	static bool CanDirectlyCast(const LogicalType &type,
+	                            const map<LogicalTypeId, CSVOption<StrpTimeFormat>> &format_options);
+
+	const idx_t scanner_idx;
+
+	//! Variable that manages buffer tracking
+	shared_ptr<CSVBufferUsage> buffer_tracker;
+
+private:
+	void Initialize() override;
+
+	void FinalizeChunkProcess() override;
+
+	//! Function used to process values that go over the first buffer, extra allocation might be necessary
+	void ProcessOverbufferValue();
+
+	void ProcessExtraRow();
+	//! Function used to move from one buffer to the other, if necessary
+	bool MoveToNextBuffer();
+
+	//! BOM skipping (https://en.wikipedia.org/wiki/Byte_order_mark)
+	void SkipBOM();
+
+	//! Skips Notes, notes are dirty lines on top of the file, before the actual data
+	void SkipCSVRows();
+
+	void SkipUntilNewLine();
+
+	void SetStart();
+
+	StringValueResult result;
+	vector<LogicalType> types;
+
+	//! Pointer to the previous buffer handle, necessary for overbuffer values
+	shared_ptr<CSVBufferHandle> previous_buffer_handle;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/planner/expression.hpp"
+#include "duckdb/planner/bound_result_modifier.hpp"
 
 namespace duckdb {
 
@@ -18,14 +19,14 @@ class PhysicalLimit : public PhysicalOperator {
 public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::LIMIT;
 
+	static constexpr const idx_t MAX_LIMIT_VALUE = 1ULL << 62ULL;
+
 public:
-	PhysicalLimit(vector<LogicalType> types, idx_t limit, idx_t offset, unique_ptr<Expression> limit_expression,
-	              unique_ptr<Expression> offset_expression, idx_t estimated_cardinality);
+	PhysicalLimit(vector<LogicalType> types, BoundLimitNode limit_val, BoundLimitNode offset_val,
+	              idx_t estimated_cardinality);
 
-	idx_t limit_value;
-	idx_t offset_value;
-	unique_ptr<Expression> limit_expression;
-	unique_ptr<Expression> offset_expression;
+	BoundLimitNode limit_val;
+	BoundLimitNode offset_val;
 
 public:
 	bool SinkOrderDependent() const override {
@@ -61,11 +62,13 @@ public:
 	}
 
 public:
-	static bool ComputeOffset(ExecutionContext &context, DataChunk &input, idx_t &limit, idx_t &offset,
-	                          idx_t current_offset, idx_t &max_element, Expression *limit_expression,
-	                          Expression *offset_expression);
+	static void SetInitialLimits(const BoundLimitNode &limit_val, const BoundLimitNode &offset_val, optional_idx &limit,
+	                             optional_idx &offset);
+	static bool ComputeOffset(ExecutionContext &context, DataChunk &input, optional_idx &limit, optional_idx &offset,
+	                          idx_t current_offset, idx_t &max_element, const BoundLimitNode &limit_val,
+	                          const BoundLimitNode &offset_val);
 	static bool HandleOffset(DataChunk &input, idx_t &current_offset, idx_t offset, idx_t limit);
-	static Value GetDelimiter(ExecutionContext &context, DataChunk &input, Expression *expr);
+	static Value GetDelimiter(ExecutionContext &context, DataChunk &input, const Expression &expr);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit_percent.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/planner/expression.hpp"
+#include "duckdb/planner/bound_result_modifier.hpp"
 
 namespace duckdb {
 
@@ -19,18 +20,11 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::LIMIT_PERCENT;
 
 public:
-	PhysicalLimitPercent(vector<LogicalType> types, double limit_percent, idx_t offset,
-	                     unique_ptr<Expression> limit_expression, unique_ptr<Expression> offset_expression,
-	                     idx_t estimated_cardinality)
-	    : PhysicalOperator(PhysicalOperatorType::LIMIT_PERCENT, std::move(types), estimated_cardinality),
-	      limit_percent(limit_percent), offset_value(offset), limit_expression(std::move(limit_expression)),
-	      offset_expression(std::move(offset_expression)) {
-	}
+	PhysicalLimitPercent(vector<LogicalType> types, BoundLimitNode limit_val_p, BoundLimitNode offset_val_p,
+	                     idx_t estimated_cardinality);
 
-	double limit_percent;
-	idx_t offset_value;
-	unique_ptr<Expression> limit_expression;
-	unique_ptr<Expression> offset_expression;
+	BoundLimitNode limit_val;
+	BoundLimitNode offset_val;
 
 public:
 	bool SinkOrderDependent() const override {

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_streaming_limit.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/planner/expression.hpp"
+#include "duckdb/planner/bound_result_modifier.hpp"
 
 namespace duckdb {
 
@@ -18,14 +19,11 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::STREAMING_LIMIT;
 
 public:
-	PhysicalStreamingLimit(vector<LogicalType> types, idx_t limit, idx_t offset,
-	                       unique_ptr<Expression> limit_expression, unique_ptr<Expression> offset_expression,
+	PhysicalStreamingLimit(vector<LogicalType> types, BoundLimitNode limit_val_p, BoundLimitNode offset_val_p,
 	                       idx_t estimated_cardinality, bool parallel);
 
-	idx_t limit_value;
-	idx_t offset_value;
-	unique_ptr<Expression> limit_expression;
-	unique_ptr<Expression> offset_expression;
+	BoundLimitNode limit_val;
+	BoundLimitNode offset_val;
 	bool parallel;
 
 public:

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_verify_vector.hpp

@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/helper/physical_verify_vector.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/physical_operator.hpp"
+#include "duckdb/planner/expression.hpp"
+
+namespace duckdb {
+
+//! The PhysicalVerifyVector operator is a streaming operator that emits the same data as it ingests - but in a
+//! different format
+// There are different configurations
+// * Dictionary: Transform every vector into a dictionary vector where the underlying vector has gaps and is reversed
+//       i.e. original: FLAT [1, 2, 3]
+//            modified: BASE: [NULL, 3, NULL, 2, NULL, 1]   OFFSETS: [5, 3, 1]
+// * Constant: Decompose every DataChunk into single-row constant vectors
+//       i.e. original: FLAT [1, 2, 3]
+//            modified: chunk #1 - CONSTANT [1]
+//                      chunk #2 - CONSTANT [2]
+//                      chunk #3 - CONSTANT [3]
+// * Sequence & Constant: Decompose every DataChunk into constant or sequence vectors based on the longest possibility
+//            original:  a: [1, 1, 20, 15, 13]   b: [1, 10, 100, 101, 102]
+//            modified:  chunk #1 - a: CONSTANT [1, 1]          b: DICTIONARY [1, 10]
+//                       chunk #2 - a: DICTIONARY [20, 15, 13]  b: SEQUENCE [100, 101, 102]
+// * Nested Shuffle: Reshuffle list vectors so that offsets are not contiguous
+//            original: [[1, 2], [3, 4]] - BASE: [1, 2, 3, 4] LISTS: [offset: 0, length: 2][offset: 2, length: 2]
+//            modified: [[1, 2], [3, 4]] - BASE: [3, 4, 1, 2] LISTS: [offset: 2, length: 2][offset: 0, length: 2]
+class PhysicalVerifyVector : public PhysicalOperator {
+public:
+	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::VERIFY_VECTOR;
+
+public:
+	explicit PhysicalVerifyVector(unique_ptr<PhysicalOperator> child);
+
+public:
+	unique_ptr<OperatorState> GetOperatorState(ExecutionContext &context) const override;
+	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
+
+	bool ParallelOperator() const override {
+		return true;
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_hash_join.hpp

@@ -52,6 +52,9 @@ public:
 	//! Used in perfect hash join
 	PerfectHashJoinStats perfect_join_statistics;
 
+public:
+	string ParamsToString() const override;
+
 public:
 	// Operator Interface
 	unique_ptr<OperatorState> GetOperatorState(ExecutionContext &context) const override;