duckdb 0.8.2-dev2399.0 → 0.8.2-dev2509.0

package/src/duckdb/src/include/duckdb/common/http_state.hpp

@@ -16,51 +16,84 @@
 
 namespace duckdb {
 
-struct CachedFile {
+class CachedFileHandle;
+
+//! Represents a file that is intended to be fully downloaded, then used in parallel by multiple threads
+class CachedFile : public std::enable_shared_from_this<CachedFile> {
+	friend class CachedFileHandle;
+
+public:
+	unique_ptr<CachedFileHandle> GetHandle() {
+		auto this_ptr = shared_from_this();
+		return make_uniq<CachedFileHandle>(this_ptr);
+	}
+
+private:
 	//! Cached Data
 	shared_ptr<char> data;
 	//! Data capacity
 	uint64_t capacity = 0;
-	//! If we finished downloading the file
-	bool finished = false;
+	//! Lock for initializing the file
+	mutex lock;
+	//! When initialized is set to true, the file is safe for parallel reading without holding the lock
+	atomic<bool> initialized = {false};
+};
+
+//! Handle to a CachedFile
+class CachedFileHandle {
+public:
+	explicit CachedFileHandle(shared_ptr<CachedFile> &file_p);
+
+	//! allocate a buffer for the file
+	void AllocateBuffer(idx_t size);
+	//! Indicate the file is fully downloaded and safe for parallel reading without lock
+	void SetInitialized();
+	//! Grow buffer to new size, copying over `bytes_to_copy` to the new buffer
+	void GrowBuffer(idx_t new_capacity, idx_t bytes_to_copy);
+	//! Write to the buffer
+	void Write(const char *buffer, idx_t length, idx_t offset = 0);
+
+	bool Initialized() {
+		return file->initialized;
+	}
+	const char *GetData() {
+		return file->data.get();
+	}
+	uint64_t GetCapacity() {
+		return file->capacity;
+	}
+
+private:
+	unique_ptr<lock_guard<mutex>> lock;
+	shared_ptr<CachedFile> file;
 };
 
 class HTTPState {
 public:
+	//! Reset all counters and cached files
+	void Reset();
+	//! Get cache entry, create if not exists
+	shared_ptr<CachedFile> &GetCachedFile(const string &path);
+	//! Helper function to get the HTTP state
+	static shared_ptr<HTTPState> TryGetState(FileOpener *opener);
+
+	bool IsEmpty() {
+		return head_count == 0 && get_count == 0 && put_count == 0 && post_count == 0 && total_bytes_received == 0 &&
+		       total_bytes_sent == 0;
+	}
+
 	atomic<idx_t> head_count {0};
 	atomic<idx_t> get_count {0};
 	atomic<idx_t> put_count {0};
 	atomic<idx_t> post_count {0};
 	atomic<idx_t> total_bytes_received {0};
 	atomic<idx_t> total_bytes_sent {0};
+
+private:
 	//! Mutex to lock when getting the cached file(Parallel Only)
 	mutex cached_files_mutex;
 	//! In case of fully downloading the file, the cached files of this query
-	unordered_map<string, CachedFile> cached_files;
-
-	void Reset() {
-		head_count = 0;
-		get_count = 0;
-		put_count = 0;
-		post_count = 0;
-		total_bytes_received = 0;
-		total_bytes_sent = 0;
-		cached_files.clear();
-	}
-
-	//! helper function to get the HTTP
-	static shared_ptr<HTTPState> TryGetState(FileOpener *opener) {
-		auto client_context = FileOpener::TryGetClientContext(opener);
-		if (client_context) {
-			return client_context->client_data->http_state;
-		}
-		return nullptr;
-	}
-
-	bool IsEmpty() {
-		return head_count == 0 && get_count == 0 && put_count == 0 && post_count == 0 && total_bytes_received == 0 &&
-		       total_bytes_sent == 0;
-	}
+	unordered_map<string, shared_ptr<CachedFile>> cached_files;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp

@@ -104,11 +104,11 @@ struct ListPackFun {
 
 struct ListSliceFun {
 	static constexpr const char *Name = "list_slice";
-	static constexpr const char *Parameters = "list,begin,end";
-	static constexpr const char *Description = "Extract a sublist using slice conventions. NULLs are interpreted as the bounds of the LIST. Negative values are accepted.";
-	static constexpr const char *Example = "list_slice(l, 2, NULL)";
+	static constexpr const char *Parameters = "list,begin,end[,step]";
+	static constexpr const char *Description = "Extract a sublist using slice conventions. Negative values are accepted.";
+	static constexpr const char *Example = "list_slice(l, 2, 4)";
 
-	static ScalarFunction GetFunction();
+	static ScalarFunctionSet GetFunctions();
 };
 
 struct ArraySliceFun {

package/src/duckdb/src/include/duckdb/parser/expression/operator_expression.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/parser/qualified_name.hpp"
+#include "duckdb/parser/expression/constant_expression.hpp"
 
 namespace duckdb {
 //! Represents a built-in operator expression
@@ -86,9 +87,25 @@ public:
 			return "(" + entry.children[0]->ToString() + " IS NOT NULL)";
 		case ExpressionType::ARRAY_EXTRACT:
 			return entry.children[0]->ToString() + "[" + entry.children[1]->ToString() + "]";
-		case ExpressionType::ARRAY_SLICE:
-			return entry.children[0]->ToString() + "[" + entry.children[1]->ToString() + ":" +
-			       entry.children[2]->ToString() + "]";
+		case ExpressionType::ARRAY_SLICE: {
+			string begin = entry.children[1]->ToString();
+			if (begin == "[]") {
+				begin = "";
+			}
+			string end = entry.children[2]->ToString();
+			if (end == "[]") {
+				if (entry.children.size() == 4) {
+					end = "-";
+				} else {
+					end = "";
+				}
+			}
+			if (entry.children.size() == 4) {
+				return entry.children[0]->ToString() + "[" + begin + ":" + end + ":" + entry.children[3]->ToString() +
+				       "]";
+			}
+			return entry.children[0]->ToString() + "[" + begin + ":" + end + "]";
+		}
 		case ExpressionType::STRUCT_EXTRACT: {
 			if (entry.children[1]->type != ExpressionType::VALUE_CONSTANT) {
 				return string();

package/src/duckdb/src/main/extension/extension_helper.cpp

@@ -180,7 +180,8 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
 #endif
 
 #ifdef DUCKDB_EXTENSIONS_TEST_WITH_LOADABLE
-	if (!initial_load && StringUtil::Contains(DUCKDB_EXTENSIONS_TEST_WITH_LOADABLE, extension)) {
+	// Note: weird comma's are on purpose to do easy string contains on a list of extension names
+	if (!initial_load && StringUtil::Contains(DUCKDB_EXTENSIONS_TEST_WITH_LOADABLE, "," + extension + ",")) {
 		Connection con(db);
 		auto result = con.Query((string) "LOAD '" + DUCKDB_EXTENSIONS_BUILD_PATH + "/" + extension + "/" + extension +
 		                        ".duckdb_extension'");

package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp

@@ -27,10 +27,19 @@ unique_ptr<ParsedExpression> Transformer::TransformArrayAccess(duckdb_libpgquery
 			children.push_back(std::move(result));
 			if (index->is_slice) {
 				// slice
-				children.push_back(!index->lidx ? make_uniq<ConstantExpression>(Value())
-				                                : TransformExpression(index->lidx));
-				children.push_back(!index->uidx ? make_uniq<ConstantExpression>(Value())
-				                                : TransformExpression(index->uidx));
+				// if either the lower or upper bound is not specified, we use an empty const list so that we can
+				// handle it in the execution
+				unique_ptr<ParsedExpression> lower =
+				    index->lidx ? TransformExpression(index->lidx)
+				                : make_uniq<ConstantExpression>(Value::LIST(LogicalType::INTEGER, vector<Value>()));
+				children.push_back(std::move(lower));
+				unique_ptr<ParsedExpression> upper =
+				    index->uidx ? TransformExpression(index->uidx)
+				                : make_uniq<ConstantExpression>(Value::LIST(LogicalType::INTEGER, vector<Value>()));
+				children.push_back(std::move(upper));
+				if (index->step) {
+					children.push_back(TransformExpression(index->step));
+				}
 				result = make_uniq<OperatorExpression>(ExpressionType::ARRAY_SLICE, std::move(children));
 			} else {
 				// array access

package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp

@@ -325,6 +325,7 @@ typedef struct PGAIndices {
 	bool is_slice; /* true if slice (i.e., colon present) */
 	PGNode *lidx;  /* slice lower bound, if any */
 	PGNode *uidx;  /* subscript, or slice upper bound if any */
+	PGNode *step;  /* slice step, if any */
 } PGAIndices;
 
 /*