duckdb 0.8.2-dev2356.0 → 0.8.2-dev2509.0

package/src/duckdb/src/include/duckdb/common/http_state.hpp

@@ -16,51 +16,84 @@
 
 namespace duckdb {
 
-struct CachedFile {
+class CachedFileHandle;
+
+//! Represents a file that is intended to be fully downloaded, then used in parallel by multiple threads
+class CachedFile : public std::enable_shared_from_this<CachedFile> {
+	friend class CachedFileHandle;
+
+public:
+	unique_ptr<CachedFileHandle> GetHandle() {
+		auto this_ptr = shared_from_this();
+		return make_uniq<CachedFileHandle>(this_ptr);
+	}
+
+private:
 	//! Cached Data
 	shared_ptr<char> data;
 	//! Data capacity
 	uint64_t capacity = 0;
-	//! If we finished downloading the file
-	bool finished = false;
+	//! Lock for initializing the file
+	mutex lock;
+	//! When initialized is set to true, the file is safe for parallel reading without holding the lock
+	atomic<bool> initialized = {false};
+};
+
+//! Handle to a CachedFile
+class CachedFileHandle {
+public:
+	explicit CachedFileHandle(shared_ptr<CachedFile> &file_p);
+
+	//! allocate a buffer for the file
+	void AllocateBuffer(idx_t size);
+	//! Indicate the file is fully downloaded and safe for parallel reading without lock
+	void SetInitialized();
+	//! Grow buffer to new size, copying over `bytes_to_copy` to the new buffer
+	void GrowBuffer(idx_t new_capacity, idx_t bytes_to_copy);
+	//! Write to the buffer
+	void Write(const char *buffer, idx_t length, idx_t offset = 0);
+
+	bool Initialized() {
+		return file->initialized;
+	}
+	const char *GetData() {
+		return file->data.get();
+	}
+	uint64_t GetCapacity() {
+		return file->capacity;
+	}
+
+private:
+	unique_ptr<lock_guard<mutex>> lock;
+	shared_ptr<CachedFile> file;
 };
 
 class HTTPState {
 public:
+	//! Reset all counters and cached files
+	void Reset();
+	//! Get cache entry, create if not exists
+	shared_ptr<CachedFile> &GetCachedFile(const string &path);
+	//! Helper function to get the HTTP state
+	static shared_ptr<HTTPState> TryGetState(FileOpener *opener);
+
+	bool IsEmpty() {
+		return head_count == 0 && get_count == 0 && put_count == 0 && post_count == 0 && total_bytes_received == 0 &&
+		       total_bytes_sent == 0;
+	}
+
 	atomic<idx_t> head_count {0};
 	atomic<idx_t> get_count {0};
 	atomic<idx_t> put_count {0};
 	atomic<idx_t> post_count {0};
 	atomic<idx_t> total_bytes_received {0};
 	atomic<idx_t> total_bytes_sent {0};
+
+private:
 	//! Mutex to lock when getting the cached file(Parallel Only)
 	mutex cached_files_mutex;
 	//! In case of fully downloading the file, the cached files of this query
-	unordered_map<string, CachedFile> cached_files;
-
-	void Reset() {
-		head_count = 0;
-		get_count = 0;
-		put_count = 0;
-		post_count = 0;
-		total_bytes_received = 0;
-		total_bytes_sent = 0;
-		cached_files.clear();
-	}
-
-	//! helper function to get the HTTP
-	static shared_ptr<HTTPState> TryGetState(FileOpener *opener) {
-		auto client_context = FileOpener::TryGetClientContext(opener);
-		if (client_context) {
-			return client_context->client_data->http_state;
-		}
-		return nullptr;
-	}
-
-	bool IsEmpty() {
-		return head_count == 0 && get_count == 0 && put_count == 0 && post_count == 0 && total_bytes_received == 0 &&
-		       total_bytes_sent == 0;
-	}
+	unordered_map<string, shared_ptr<CachedFile>> cached_files;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp

@@ -22,14 +22,14 @@ struct ListSegment {
 	ListSegment *next;
 };
 struct LinkedList {
-	LinkedList() {};
+	LinkedList() : total_capacity(0), first_segment(nullptr), last_segment(nullptr) {};
 	LinkedList(idx_t total_capacity_p, ListSegment *first_segment_p, ListSegment *last_segment_p)
 	    : total_capacity(total_capacity_p), first_segment(first_segment_p), last_segment(last_segment_p) {
 	}
 
-	idx_t total_capacity = 0;
-	ListSegment *first_segment = nullptr;
-	ListSegment *last_segment = nullptr;
+	idx_t total_capacity;
+	ListSegment *first_segment;
+	ListSegment *last_segment;
 };
 
 // forward declarations
@@ -37,23 +37,21 @@ struct ListSegmentFunctions;
 typedef ListSegment *(*create_segment_t)(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
                                          uint16_t capacity);
 typedef void (*write_data_to_segment_t)(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
-                                        ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count);
+                                        ListSegment *segment, RecursiveUnifiedVectorFormat &input_data,
+                                        idx_t &entry_idx);
 typedef void (*read_data_from_segment_t)(const ListSegmentFunctions &functions, const ListSegment *segment,
                                          Vector &result, idx_t &total_count);
-typedef ListSegment *(*copy_data_from_segment_t)(const ListSegmentFunctions &functions, const ListSegment *source,
-                                                 ArenaAllocator &allocator);
 
 struct ListSegmentFunctions {
 	create_segment_t create_segment;
 	write_data_to_segment_t write_data;
 	read_data_from_segment_t read_data;
-	copy_data_from_segment_t copy_data;
+
 	vector<ListSegmentFunctions> child_functions;
 
-	void AppendRow(ArenaAllocator &allocator, LinkedList &linked_list, Vector &input, idx_t &entry_idx,
-	               idx_t &count) const;
+	void AppendRow(ArenaAllocator &allocator, LinkedList &linked_list, RecursiveUnifiedVectorFormat &input_data,
+	               idx_t &entry_idx) const;
 	void BuildListVector(const LinkedList &linked_list, Vector &result, idx_t &initial_total_count) const;
-	void CopyLinkedList(const LinkedList &source_list, LinkedList &target_list, ArenaAllocator &allocator) const;
 };
 
 void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType &type);

package/src/duckdb/src/include/duckdb/common/types/vector.hpp

@@ -35,6 +35,11 @@ struct UnifiedVectorFormat {
 	}
 };
 
+struct RecursiveUnifiedVectorFormat {
+	UnifiedVectorFormat unified;
+	vector<RecursiveUnifiedVectorFormat> children;
+};
+
 class VectorCache;
 class VectorStructBuffer;
 class VectorListBuffer;
@@ -140,6 +145,8 @@ public:
 	//! The most common vector types (flat, constant & dictionary) can be converted to the canonical format "for free"
 	//! ToUnifiedFormat was originally called Orrify, as a tribute to Orri Erling who came up with it
 	DUCKDB_API void ToUnifiedFormat(idx_t count, UnifiedVectorFormat &data);
+	//! Recursively calls UnifiedVectorFormat on a vector and its child vectors (for nested types)
+	static void RecursiveToUnifiedFormat(Vector &input, idx_t count, RecursiveUnifiedVectorFormat &data);
 
 	//! Turn the vector into a sequence vector
 	DUCKDB_API void Sequence(int64_t start, int64_t increment, idx_t count);

package/src/duckdb/src/include/duckdb/common/vector_operations/aggregate_executor.hpp

@@ -15,9 +15,14 @@
 
 namespace duckdb {
 
+// structs
 struct AggregateInputData;
-
-typedef std::pair<idx_t, idx_t> FrameBounds;
+struct FrameBounds {
+	FrameBounds() : start(0), end(0) {};
+	FrameBounds(idx_t start, idx_t end) : start(start), end(end) {};
+	idx_t start = 0;
+	idx_t end = 0;
+};
 
 class AggregateExecutor {
 private:

package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp

@@ -104,11 +104,11 @@ struct ListPackFun {
 
 struct ListSliceFun {
 	static constexpr const char *Name = "list_slice";
-	static constexpr const char *Parameters = "list,begin,end";
-	static constexpr const char *Description = "Extract a sublist using slice conventions. NULLs are interpreted as the bounds of the LIST. Negative values are accepted.";
-	static constexpr const char *Example = "list_slice(l, 2, NULL)";
+	static constexpr const char *Parameters = "list,begin,end[,step]";
+	static constexpr const char *Description = "Extract a sublist using slice conventions. Negative values are accepted.";
+	static constexpr const char *Example = "list_slice(l, 2, 4)";
 
-	static ScalarFunction GetFunction();
+	static ScalarFunctionSet GetFunctions();
 };
 
 struct ArraySliceFun {

package/src/duckdb/src/include/duckdb/execution/perfect_aggregate_hashtable.hpp

@@ -56,8 +56,10 @@ protected:
 	//! Reused selection vector
 	SelectionVector sel;
 
-	//! The arena allocator used by the aggregates for their internal state
-	ArenaAllocator aggregate_allocator;
+	//! The active arena allocator used by the aggregates for their internal state
+	unique_ptr<ArenaAllocator> aggregate_allocator;
+	//! Owning arena allocators that this HT has data from
+	vector<unique_ptr<ArenaAllocator>> stored_allocators;
 
 private:
 	//! Destroy the perfect aggregate HT (called automatically by the destructor)

package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp

@@ -117,8 +117,6 @@ public:
 
 class WindowSegmentTree : public WindowAggregator {
 public:
-	using FrameBounds = std::pair<idx_t, idx_t>;
-
 	WindowSegmentTree(AggregateObject aggr, const LogicalType &result_type, idx_t count, WindowAggregationMode mode_p);
 	~WindowSegmentTree() override;
 

package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp

@@ -41,7 +41,6 @@ typedef void (*aggregate_simple_update_t)(Vector inputs[], AggregateInputData &a
                                           data_ptr_t state, idx_t count);
 
 //! The type used for updating complex windowed aggregate functions (optional)
-typedef std::pair<idx_t, idx_t> FrameBounds;
 typedef void (*aggregate_window_t)(Vector inputs[], const ValidityMask &filter_mask,
                                    AggregateInputData &aggr_input_data, idx_t input_count, data_ptr_t state,
                                    const FrameBounds &frame, const FrameBounds &prev, Vector &result, idx_t rid,

package/src/duckdb/src/include/duckdb/parser/expression/operator_expression.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/parser/qualified_name.hpp"
+#include "duckdb/parser/expression/constant_expression.hpp"
 
 namespace duckdb {
 //! Represents a built-in operator expression
@@ -86,9 +87,25 @@ public:
 			return "(" + entry.children[0]->ToString() + " IS NOT NULL)";
 		case ExpressionType::ARRAY_EXTRACT:
 			return entry.children[0]->ToString() + "[" + entry.children[1]->ToString() + "]";
-		case ExpressionType::ARRAY_SLICE:
-			return entry.children[0]->ToString() + "[" + entry.children[1]->ToString() + ":" +
-			       entry.children[2]->ToString() + "]";
+		case ExpressionType::ARRAY_SLICE: {
+			string begin = entry.children[1]->ToString();
+			if (begin == "[]") {
+				begin = "";
+			}
+			string end = entry.children[2]->ToString();
+			if (end == "[]") {
+				if (entry.children.size() == 4) {
+					end = "-";
+				} else {
+					end = "";
+				}
+			}
+			if (entry.children.size() == 4) {
+				return entry.children[0]->ToString() + "[" + begin + ":" + end + ":" + entry.children[3]->ToString() +
+				       "]";
+			}
+			return entry.children[0]->ToString() + "[" + begin + ":" + end + "]";
+		}
 		case ExpressionType::STRUCT_EXTRACT: {
 			if (entry.children[1]->type != ExpressionType::VALUE_CONSTANT) {
 				return string();

package/src/duckdb/src/main/extension/extension_helper.cpp

@@ -180,7 +180,8 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
 #endif
 
 #ifdef DUCKDB_EXTENSIONS_TEST_WITH_LOADABLE
-	if (!initial_load && StringUtil::Contains(DUCKDB_EXTENSIONS_TEST_WITH_LOADABLE, extension)) {
+	// Note: weird comma's are on purpose to do easy string contains on a list of extension names
+	if (!initial_load && StringUtil::Contains(DUCKDB_EXTENSIONS_TEST_WITH_LOADABLE, "," + extension + ",")) {
 		Connection con(db);
 		auto result = con.Query((string) "LOAD '" + DUCKDB_EXTENSIONS_BUILD_PATH + "/" + extension + "/" + extension +
 		                        ".duckdb_extension'");

package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp

@@ -27,10 +27,19 @@ unique_ptr<ParsedExpression> Transformer::TransformArrayAccess(duckdb_libpgquery
 			children.push_back(std::move(result));
 			if (index->is_slice) {
 				// slice
-				children.push_back(!index->lidx ? make_uniq<ConstantExpression>(Value())
-				                                : TransformExpression(index->lidx));
-				children.push_back(!index->uidx ? make_uniq<ConstantExpression>(Value())
-				                                : TransformExpression(index->uidx));
+				// if either the lower or upper bound is not specified, we use an empty const list so that we can
+				// handle it in the execution
+				unique_ptr<ParsedExpression> lower =
+				    index->lidx ? TransformExpression(index->lidx)
+				                : make_uniq<ConstantExpression>(Value::LIST(LogicalType::INTEGER, vector<Value>()));
+				children.push_back(std::move(lower));
+				unique_ptr<ParsedExpression> upper =
+				    index->uidx ? TransformExpression(index->uidx)
+				                : make_uniq<ConstantExpression>(Value::LIST(LogicalType::INTEGER, vector<Value>()));
+				children.push_back(std::move(upper));
+				if (index->step) {
+					children.push_back(TransformExpression(index->step));
+				}
 				result = make_uniq<OperatorExpression>(ExpressionType::ARRAY_SLICE, std::move(children));
 			} else {
 				// array access

package/src/duckdb/src/parser/transform/expression/transform_function.cpp

@@ -209,6 +209,9 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 			}
 			window_ref = it->second;
 			D_ASSERT(window_ref);
+			if (window_ref->startOffset || window_ref->endOffset || window_ref->frameOptions != FRAMEOPTION_DEFAULTS) {
+				throw ParserException("cannot copy window \"%s\" because it has a frame clause", window_spec->refname);
+			}
 		}
 		in_window_definition = true;
 		TransformWindowDef(*window_ref, *expr);

package/src/duckdb/src/storage/serialization/serialize_constraint.cpp

@@ -50,7 +50,7 @@ void ForeignKeyConstraint::FormatSerialize(FormatSerializer &serializer) const {
 	Constraint::FormatSerialize(serializer);
 	serializer.WriteProperty("pk_columns", pk_columns);
 	serializer.WriteProperty("fk_columns", fk_columns);
-	serializer.WriteProperty("type", info.type);
+	serializer.WriteProperty("fk_type", info.type);
 	serializer.WriteProperty("schema", info.schema);
 	serializer.WriteProperty("table", info.table);
 	serializer.WriteProperty("pk_keys", info.pk_keys);
@@ -61,7 +61,7 @@ unique_ptr<Constraint> ForeignKeyConstraint::FormatDeserialize(FormatDeserialize
 	auto result = duckdb::unique_ptr<ForeignKeyConstraint>(new ForeignKeyConstraint());
 	deserializer.ReadProperty("pk_columns", result->pk_columns);
 	deserializer.ReadProperty("fk_columns", result->fk_columns);
-	deserializer.ReadProperty("type", result->info.type);
+	deserializer.ReadProperty("fk_type", result->info.type);
 	deserializer.ReadProperty("schema", result->info.schema);
 	deserializer.ReadProperty("table", result->info.table);
 	deserializer.ReadProperty("pk_keys", result->info.pk_keys);

package/src/duckdb/src/storage/serialization/serialize_create_info.cpp

@@ -160,13 +160,13 @@ unique_ptr<CreateInfo> CreateTableInfo::FormatDeserialize(FormatDeserializer &de
 void CreateTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
 	CreateInfo::FormatSerialize(serializer);
 	serializer.WriteProperty("name", name);
-	serializer.WriteProperty("type", type);
+	serializer.WriteProperty("logical_type", type);
 }
 
 unique_ptr<CreateInfo> CreateTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
 	auto result = duckdb::unique_ptr<CreateTypeInfo>(new CreateTypeInfo());
 	deserializer.ReadProperty("name", result->name);
-	deserializer.ReadProperty("type", result->type);
+	deserializer.ReadProperty("logical_type", result->type);
 	return std::move(result);
 }
 

package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp

@@ -176,7 +176,7 @@ void AlterForeignKeyInfo::FormatSerialize(FormatSerializer &serializer) const {
 	serializer.WriteProperty("fk_columns", fk_columns);
 	serializer.WriteProperty("pk_keys", pk_keys);
 	serializer.WriteProperty("fk_keys", fk_keys);
-	serializer.WriteProperty("type", type);
+	serializer.WriteProperty("alter_fk_type", type);
 }
 
 unique_ptr<AlterTableInfo> AlterForeignKeyInfo::FormatDeserialize(FormatDeserializer &deserializer) {
@@ -186,7 +186,7 @@ unique_ptr<AlterTableInfo> AlterForeignKeyInfo::FormatDeserialize(FormatDeserial
 	deserializer.ReadProperty("fk_columns", result->fk_columns);
 	deserializer.ReadProperty("pk_keys", result->pk_keys);
 	deserializer.ReadProperty("fk_keys", result->fk_keys);
-	deserializer.ReadProperty("type", result->type);
+	deserializer.ReadProperty("alter_fk_type", result->type);
 	return std::move(result);
 }
 

package/src/duckdb/src/storage/serialization/serialize_tableref.cpp

@@ -96,7 +96,7 @@ void JoinRef::FormatSerialize(FormatSerializer &serializer) const {
 	serializer.WriteProperty("left", *left);
 	serializer.WriteProperty("right", *right);
 	serializer.WriteOptionalProperty("condition", condition);
-	serializer.WriteProperty("type", type);
+	serializer.WriteProperty("join_type", type);
 	serializer.WriteProperty("ref_type", ref_type);
 	serializer.WriteProperty("using_columns", using_columns);
 }
@@ -106,7 +106,7 @@ unique_ptr<TableRef> JoinRef::FormatDeserialize(FormatDeserializer &deserializer
 	deserializer.ReadProperty("left", result->left);
 	deserializer.ReadProperty("right", result->right);
 	deserializer.ReadOptionalProperty("condition", result->condition);
-	deserializer.ReadProperty("type", result->type);
+	deserializer.ReadProperty("join_type", result->type);
 	deserializer.ReadProperty("ref_type", result->ref_type);
 	deserializer.ReadProperty("using_columns", result->using_columns);
 	return std::move(result);
@@ -151,14 +151,12 @@ unique_ptr<TableRef> SubqueryRef::FormatDeserialize(FormatDeserializer &deserial
 void TableFunctionRef::FormatSerialize(FormatSerializer &serializer) const {
 	TableRef::FormatSerialize(serializer);
 	serializer.WriteProperty("function", *function);
-	serializer.WriteProperty("alias", alias);
 	serializer.WriteProperty("column_name_alias", column_name_alias);
 }
 
 unique_ptr<TableRef> TableFunctionRef::FormatDeserialize(FormatDeserializer &deserializer) {
 	auto result = duckdb::unique_ptr<TableFunctionRef>(new TableFunctionRef());
 	deserializer.ReadProperty("function", result->function);
-	deserializer.ReadProperty("alias", result->alias);
 	deserializer.ReadProperty("column_name_alias", result->column_name_alias);
 	return std::move(result);
 }

package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp

@@ -325,6 +325,7 @@ typedef struct PGAIndices {
 	bool is_slice; /* true if slice (i.e., colon present) */
 	PGNode *lidx;  /* slice lower bound, if any */
 	PGNode *uidx;  /* subscript, or slice upper bound if any */
+	PGNode *step;  /* slice step, if any */
 } PGAIndices;
 
 /*