duckdb 0.3.2-dev502.0 → 0.3.2-dev528.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.2-dev502.0",
+  "version": "0.3.2-dev528.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -11770,6 +11770,8 @@ string Exception::ExceptionTypeToString(ExceptionType type) {
 		return "Invalid Input";
 	case ExceptionType::OUT_OF_MEMORY:
 		return "Out of Memory";
+	case ExceptionType::PERMISSION:
+		return "Permission";
 	default:
 		return "Unknown";
 	}
@@ -11855,6 +11857,9 @@ CatalogException::CatalogException(const string &msg) : StandardException(Except
 ParserException::ParserException(const string &msg) : StandardException(ExceptionType::PARSER, msg) {
 }
 
+PermissionException::PermissionException(const string &msg) : StandardException(ExceptionType::PERMISSION, msg) {
+}
+
 SyntaxException::SyntaxException(const string &msg) : Exception(ExceptionType::SYNTAX, msg) {
 }
 
@@ -27293,11 +27298,9 @@ struct RowOperations {
 	//! Swizzles the base pointer of each row to offset within heap block
 	static void SwizzleHeapPointer(const RowLayout &layout, data_ptr_t row_ptr, const data_ptr_t heap_base_ptr,
 	                               const idx_t count);
-	//! Swizzles the base offset of each row back to a pointer
-	static void UnswizzleHeapPointer(const RowLayout &layout, data_ptr_t row_ptr, const data_ptr_t heap_base_ptr,
-	                                 const idx_t count);
-	//! Unswizzles offsets back to pointers to blobs
-	static void UnswizzleColumns(const RowLayout &layout, const data_ptr_t base_row_ptr, const idx_t count);
+	//! Unswizzles all offsets back to pointers
+	static void UnswizzlePointers(const RowLayout &layout, const data_ptr_t base_row_ptr,
+	                              const data_ptr_t base_heap_ptr, const idx_t count);
 };
 
 } // namespace duckdb
@@ -27533,46 +27536,44 @@ namespace duckdb {
 
 void RowOperations::SwizzleColumns(const RowLayout &layout, const data_ptr_t base_row_ptr, const idx_t count) {
 	const idx_t row_width = layout.GetRowWidth();
-	const idx_t heap_pointer_offset = layout.GetHeapPointerOffset();
-	// Swizzle the blob columns one by one
-	for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) {
-		auto physical_type = layout.GetTypes()[col_idx].InternalType();
-		if (TypeIsConstantSize(physical_type)) {
-			continue;
+	data_ptr_t heap_row_ptrs[STANDARD_VECTOR_SIZE];
+	idx_t done = 0;
+	while (done != count) {
+		const idx_t next = MinValue<idx_t>(count - done, STANDARD_VECTOR_SIZE);
+		const data_ptr_t row_ptr = base_row_ptr + done * row_width;
+		// Load heap row pointers
+		data_ptr_t heap_ptr_ptr = row_ptr + layout.GetHeapPointerOffset();
+		for (idx_t i = 0; i < next; i++) {
+			heap_row_ptrs[i] = Load<data_ptr_t>(heap_ptr_ptr);
+			heap_ptr_ptr += row_width;
 		}
-		data_ptr_t row_ptr = base_row_ptr;
-		const idx_t &col_offset = layout.GetOffsets()[col_idx];
-		if (physical_type == PhysicalType::VARCHAR) {
-			// Replace the pointer with the computed offset (if non-inlined)
-			const idx_t string_pointer_offset = sizeof(uint32_t) + string_t::PREFIX_LENGTH;
-			for (idx_t i = 0; i < count; i++) {
-				const string_t str = Load<string_t>(row_ptr + col_offset);
-				if (!str.IsInlined()) {
-					// Load the pointer to the start of the row in the heap
-					data_ptr_t heap_row_ptr = Load<data_ptr_t>(row_ptr + heap_pointer_offset);
-					// This is where the pointer that points to the heap is stored in the RowLayout
-					data_ptr_t col_ptr = row_ptr + col_offset + string_pointer_offset;
-					// Load the pointer to the data of this column in the same heap row
-					data_ptr_t heap_col_ptr = Load<data_ptr_t>(col_ptr);
-					// Overwrite the column data pointer with the within-row offset (pointer arithmetic)
-					Store<idx_t>(heap_col_ptr - heap_row_ptr, col_ptr);
-				}
-				row_ptr += row_width;
+		// Loop through the blob columns
+		for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) {
+			auto physical_type = layout.GetTypes()[col_idx].InternalType();
+			if (TypeIsConstantSize(physical_type)) {
+				continue;
 			}
-		} else {
-			// Replace the pointer with the computed offset
-			for (idx_t i = 0; i < count; i++) {
-				// Load the pointer to the start of the row in the heap
-				data_ptr_t heap_row_ptr = Load<data_ptr_t>(row_ptr + heap_pointer_offset);
-				// This is where the pointer that points to the heap is stored in the RowLayout
-				data_ptr_t col_ptr = row_ptr + col_offset;
-				// Load the pointer to the data of this column in the same heap row
-				data_ptr_t heap_col_ptr = Load<data_ptr_t>(col_ptr);
-				// Overwrite the column data pointer with the within-row offset (pointer arithmetic)
-				Store<idx_t>(heap_col_ptr - heap_row_ptr, col_ptr);
-				row_ptr += row_width;
+			data_ptr_t col_ptr = row_ptr + layout.GetOffsets()[col_idx];
+			if (physical_type == PhysicalType::VARCHAR) {
+				data_ptr_t string_ptr = col_ptr + sizeof(uint32_t) + string_t::PREFIX_LENGTH;
+				for (idx_t i = 0; i < next; i++) {
+					if (Load<uint32_t>(col_ptr) > string_t::INLINE_LENGTH) {
+						// Overwrite the string pointer with the within-row offset (if not inlined)
+						Store<idx_t>(Load<data_ptr_t>(string_ptr) - heap_row_ptrs[i], string_ptr);
+					}
+					col_ptr += row_width;
+					string_ptr += row_width;
+				}
+			} else {
+				// Non-varchar blob columns
+				for (idx_t i = 0; i < next; i++) {
+					// Overwrite the column data pointer with the within-row offset
+					Store<idx_t>(Load<data_ptr_t>(col_ptr) - heap_row_ptrs[i], col_ptr);
+					col_ptr += row_width;
+				}
 			}
 		}
+		done += next;
 	}
 }
 
@@ -27588,59 +27589,48 @@ void RowOperations::SwizzleHeapPointer(const RowLayout &layout, data_ptr_t row_p
 	}
 }
 
-void RowOperations::UnswizzleHeapPointer(const RowLayout &layout, data_ptr_t row_ptr, const data_ptr_t heap_base_ptr,
-                                         const idx_t count) {
+void RowOperations::UnswizzlePointers(const RowLayout &layout, const data_ptr_t base_row_ptr,
+                                      const data_ptr_t base_heap_ptr, const idx_t count) {
 	const idx_t row_width = layout.GetRowWidth();
-	row_ptr += layout.GetHeapPointerOffset();
-	for (idx_t i = 0; i < count; i++) {
-		idx_t heap_row_offset = Load<idx_t>(row_ptr);
-		Store<data_ptr_t>(heap_base_ptr + heap_row_offset, row_ptr);
-		row_ptr += row_width;
-	}
-}
-
-void RowOperations::UnswizzleColumns(const RowLayout &layout, const data_ptr_t base_row_ptr, const idx_t count) {
-	const idx_t row_width = layout.GetRowWidth();
-	const idx_t heap_pointer_offset = layout.GetHeapPointerOffset();
-	// Unswizzle the columns one by one
-	for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) {
-		auto physical_type = layout.GetTypes()[col_idx].InternalType();
-		if (TypeIsConstantSize(physical_type)) {
-			continue;
-		}
-		const idx_t col_offset = layout.GetOffsets()[col_idx];
-		data_ptr_t row_ptr = base_row_ptr;
-		if (physical_type == PhysicalType::VARCHAR) {
-			// Replace offset with the pointer (if non-inlined)
-			const idx_t string_pointer_offset = sizeof(uint32_t) + string_t::PREFIX_LENGTH;
-			for (idx_t i = 0; i < count; i++) {
-				const string_t str = Load<string_t>(row_ptr + col_offset);
-				if (!str.IsInlined()) {
-					// Load the pointer to the start of the row in the heap
-					data_ptr_t heap_row_ptr = Load<data_ptr_t>(row_ptr + heap_pointer_offset);
-					// This is where the pointer that points to the heap is stored in the RowLayout
-					data_ptr_t col_ptr = row_ptr + col_offset + string_pointer_offset;
-					// Load the offset to the data of this column in the same heap row
-					idx_t heap_col_offset = Load<idx_t>(col_ptr);
+	data_ptr_t heap_row_ptrs[STANDARD_VECTOR_SIZE];
+	idx_t done = 0;
+	while (done != count) {
+		const idx_t next = MinValue<idx_t>(count - done, STANDARD_VECTOR_SIZE);
+		const data_ptr_t row_ptr = base_row_ptr + done * row_width;
+		// Restore heap row pointers
+		data_ptr_t heap_ptr_ptr = row_ptr + layout.GetHeapPointerOffset();
+		for (idx_t i = 0; i < next; i++) {
+			heap_row_ptrs[i] = base_heap_ptr + Load<idx_t>(heap_ptr_ptr);
+			Store<data_ptr_t>(heap_row_ptrs[i], heap_ptr_ptr);
+			heap_ptr_ptr += row_width;
+		}
+		// Loop through the blob columns
+		for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) {
+			auto physical_type = layout.GetTypes()[col_idx].InternalType();
+			if (TypeIsConstantSize(physical_type)) {
+				continue;
+			}
+			data_ptr_t col_ptr = row_ptr + layout.GetOffsets()[col_idx];
+			if (physical_type == PhysicalType::VARCHAR) {
+				data_ptr_t string_ptr = col_ptr + sizeof(uint32_t) + string_t::PREFIX_LENGTH;
+				for (idx_t i = 0; i < next; i++) {
+					if (Load<uint32_t>(col_ptr) > string_t::INLINE_LENGTH) {
+						// Overwrite the string offset with the pointer (if not inlined)
+						Store<data_ptr_t>(heap_row_ptrs[i] + Load<idx_t>(string_ptr), string_ptr);
+					}
+					col_ptr += row_width;
+					string_ptr += row_width;
+				}
+			} else {
+				// Non-varchar blob columns
+				for (idx_t i = 0; i < next; i++) {
 					// Overwrite the column data offset with the pointer
-					Store<data_ptr_t>(heap_row_ptr + heap_col_offset, col_ptr);
+					Store<data_ptr_t>(heap_row_ptrs[i] + Load<idx_t>(col_ptr), col_ptr);
+					col_ptr += row_width;
 				}
-				row_ptr += row_width;
-			}
-		} else {
-			// Replace the offset with the pointer
-			for (idx_t i = 0; i < count; i++) {
-				// Load the pointer to the start of the row in the heap
-				data_ptr_t heap_row_ptr = Load<data_ptr_t>(row_ptr + heap_pointer_offset);
-				// This is where the pointer that points to the heap is stored in the RowLayout
-				data_ptr_t col_ptr = row_ptr + col_offset;
-				// Load the offset to the data of this column in the same heap row
-				idx_t heap_col_offset = Load<idx_t>(col_ptr);
-				// Overwrite the column data offset with the pointer
-				Store<data_ptr_t>(heap_row_ptr + heap_col_offset, col_ptr);
-				row_ptr += row_width;
 			}
 		}
+		done += next;
 	}
 }
 
@@ -32711,8 +32701,7 @@ void SortedData::Unswizzle() {
 		auto &heap_block = heap_blocks[i];
 		auto data_handle_p = buffer_manager.Pin(data_block.block);
 		auto heap_handle_p = buffer_manager.Pin(heap_block.block);
-		RowOperations::UnswizzleHeapPointer(layout, data_handle_p->Ptr(), heap_handle_p->Ptr(), data_block.count);
-		RowOperations::UnswizzleColumns(layout, data_handle_p->Ptr(), data_block.count);
+		RowOperations::UnswizzlePointers(layout, data_handle_p->Ptr(), heap_handle_p->Ptr(), data_block.count);
 		state.heap_blocks.push_back(move(heap_block));
 		state.pinned_blocks.push_back(move(heap_handle_p));
 	}
@@ -32958,9 +32947,7 @@ void PayloadScanner::Scan(DataChunk &chunk) {
 		}
 		// Unswizzle the offsets back to pointers (if needed)
 		if (!sorted_data.layout.AllConstant() && global_sort_state.external) {
-			RowOperations::UnswizzleHeapPointer(sorted_data.layout, data_ptr, read_state.payload_heap_handle->Ptr(),
-			                                    next);
-			RowOperations::UnswizzleColumns(sorted_data.layout, data_ptr, next);
+			RowOperations::UnswizzlePointers(sorted_data.layout, data_ptr, read_state.payload_heap_handle->Ptr(), next);
 		}
 		// Update state indices
 		read_state.entry_idx += next;
@@ -38105,7 +38092,7 @@ bool TryAddOperator::Operation(int16_t left, int16_t right, int16_t &result);
 template <>
 bool TryAddOperator::Operation(int32_t left, int32_t right, int32_t &result);
 template <>
-bool TryAddOperator::Operation(int64_t left, int64_t right, int64_t &result);
+DUCKDB_API bool TryAddOperator::Operation(int64_t left, int64_t right, int64_t &result);
 
 struct AddOperatorOverflowCheck {
 	template <class TA, class TB, class TR>
@@ -38217,7 +38204,7 @@ bool TryMultiplyOperator::Operation(int16_t left, int16_t right, int16_t &result
 template <>
 bool TryMultiplyOperator::Operation(int32_t left, int32_t right, int32_t &result);
 template <>
-bool TryMultiplyOperator::Operation(int64_t left, int64_t right, int64_t &result);
+DUCKDB_API bool TryMultiplyOperator::Operation(int64_t left, int64_t right, int64_t &result);
 
 struct MultiplyOperatorOverflowCheck {
 	template <class TA, class TB, class TR>
@@ -39497,7 +39484,12 @@ timestamp_t Timestamp::FromDatetime(date_t date, dtime_t time) {
 
 void Timestamp::Convert(timestamp_t timestamp, date_t &out_date, dtime_t &out_time) {
 	out_date = GetDate(timestamp);
-	out_time = dtime_t(timestamp.value - (int64_t(out_date.days) * int64_t(Interval::MICROS_PER_DAY)));
+	int64_t days_micros;
+	if (!TryMultiplyOperator::Operation<int64_t, int64_t, int64_t>(out_date.days, Interval::MICROS_PER_DAY,
+	                                                               days_micros)) {
+		throw ConversionException("Date out of range in timestamp conversion");
+	}
+	out_time = dtime_t(timestamp.value - days_micros);
 	D_ASSERT(timestamp == Timestamp::FromDatetime(out_date, out_time));
 }
 
@@ -69880,9 +69872,14 @@ protected:
 } // namespace duckdb
 
 
+
 namespace duckdb {
 
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalExport &op) {
+	auto &config = DBConfig::GetConfig(context);
+	if (!config.enable_external_access) {
+		throw PermissionException("Export is disabled through configuration");
+	}
 	auto export_node = make_unique<PhysicalExport>(op.types, op.function, move(op.copy_info), op.estimated_cardinality,
 	                                               op.exported_tables);
 	// plan the underlying copy statements, if any
@@ -81429,6 +81426,7 @@ void PragmaFunctions::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+
 namespace duckdb {
 
 string PragmaTableInfo(ClientContext &context, const FunctionParameters &parameters) {
@@ -81469,6 +81467,10 @@ string PragmaVersion(ClientContext &context, const FunctionParameters &parameter
 }
 
 string PragmaImportDatabase(ClientContext &context, const FunctionParameters &parameters) {
+	auto &config = DBConfig::GetConfig(context);
+	if (!config.enable_external_access) {
+		throw PermissionException("Import is disabled through configuration");
+	}
 	auto &fs = FileSystem::GetFileSystem(context);
 	auto *opener = FileSystem::GetFileOpener(context);
 
@@ -97079,6 +97081,7 @@ void CSVCopyFunction::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+
 namespace duckdb {
 
 struct GlobFunctionBindData : public TableFunctionData {
@@ -97090,6 +97093,10 @@ static unique_ptr<FunctionData> GlobFunctionBind(ClientContext &context, vector<
                                                  vector<LogicalType> &input_table_types,
                                                  vector<string> &input_table_names, vector<LogicalType> &return_types,
                                                  vector<string> &names) {
+	auto &config = DBConfig::GetConfig(context);
+	if (!config.enable_external_access) {
+		throw PermissionException("Globbing is disabled through configuration");
+	}
 	auto result = make_unique<GlobFunctionBindData>();
 	auto &fs = FileSystem::GetFileSystem(context);
 	result->files = fs.Glob(inputs[0].str_value);
@@ -97880,6 +97887,10 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, vector<Value
                                             unordered_map<string, Value> &named_parameters,
                                             vector<LogicalType> &input_table_types, vector<string> &input_table_names,
                                             vector<LogicalType> &return_types, vector<string> &names) {
+	auto &config = DBConfig::GetConfig(context);
+	if (!config.enable_external_access) {
+		throw PermissionException("Scanning CSV files is disabled through configuration");
+	}
 	auto result = make_unique<ReadCSVData>();
 	auto &options = result->options;
 
@@ -113737,7 +113748,7 @@ const vector<string> ExtensionHelper::PATH_COMPONENTS = {".duckdb", "extensions"
 void ExtensionHelper::InstallExtension(DatabaseInstance &db, const string &extension, bool force_install) {
 	auto &config = DBConfig::GetConfig(db);
 	if (!config.enable_external_access) {
-		throw Exception("Installing extensions is disabled");
+		throw PermissionException("Installing extensions is disabled through configuration");
 	}
 	auto &fs = FileSystem::GetFileSystem(db);
 
@@ -113876,7 +113887,7 @@ static T LoadFunctionFromDLL(void *dll, const string &function_name, const strin
 void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, const string &extension) {
 	auto &config = DBConfig::GetConfig(db);
 	if (!config.enable_external_access) {
-		throw Exception("Loading external extensions is disabled");
+		throw PermissionException("Loading external extensions is disabled through configuration");
 	}
 	auto &fs = FileSystem::GetFileSystem(db);
 	auto filename = fs.ConvertSeparators(extension);
@@ -117750,10 +117761,11 @@ Value DisabledOptimizersSetting::GetSetting(ClientContext &context) {
 // Enable External Access
 //===--------------------------------------------------------------------===//
 void EnableExternalAccessSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
-	if (db) {
+	auto new_value = input.GetValue<bool>();
+	if (db && new_value) {
 		throw InvalidInputException("Cannot change enable_external_access setting while database is running");
 	}
-	config.enable_external_access = input.GetValue<bool>();
+	config.enable_external_access = new_value;
 }
 
 Value EnableExternalAccessSetting::GetSetting(ClientContext &context) {
@@ -150187,7 +150199,8 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundRecursiveCTENode &node) {
 	left_node = CastLogicalOperatorToTypes(node.left->types, node.types, move(left_node));
 	right_node = CastLogicalOperatorToTypes(node.right->types, node.types, move(right_node));
 
-	if (node.right_binder->bind_context.cte_references[node.ctename] == nullptr) {
+	if (!node.right_binder->bind_context.cte_references[node.ctename] ||
+	    *node.right_binder->bind_context.cte_references[node.ctename] == 0) {
 		auto root = make_unique<LogicalSetOperation>(node.setop_index, node.types.size(), move(left_node),
 		                                             move(right_node), LogicalOperatorType::LOGICAL_UNION);
 		return VisitQueryNode(node, move(root));
@@ -150914,7 +150927,7 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 	// COPY TO a file
 	auto &config = DBConfig::GetConfig(context);
 	if (!config.enable_external_access) {
-		throw Exception("COPY TO is disabled by configuration");
+		throw PermissionException("COPY TO is disabled by configuration");
 	}
 	BoundStatement result;
 	result.types = {LogicalType::BIGINT};
@@ -150944,7 +150957,7 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 BoundStatement Binder::BindCopyFrom(CopyStatement &stmt) {
 	auto &config = DBConfig::GetConfig(context);
 	if (!config.enable_external_access) {
-		throw Exception("COPY FROM is disabled by configuration");
+		throw PermissionException("COPY FROM is disabled by configuration");
 	}
 	BoundStatement result;
 	result.types = {LogicalType::BIGINT};
@@ -151776,7 +151789,7 @@ BoundStatement Binder::Bind(ExportStatement &stmt) {
 	// COPY TO a file
 	auto &config = DBConfig::GetConfig(context);
 	if (!config.enable_external_access) {
-		throw Exception("COPY TO is disabled by configuration");
+		throw PermissionException("COPY TO is disabled through configuration");
 	}
 	BoundStatement result;
 	result.types = {LogicalType::BOOLEAN};

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "7a4c1fbf0"
-#define DUCKDB_VERSION "0.3.2-dev502"
+#define DUCKDB_SOURCE_ID "0546552f6"
+#define DUCKDB_VERSION "0.3.2-dev528"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -1146,7 +1146,7 @@ bool IsValidType() {
 extern const LogicalType LOGICAL_ROW_TYPE;
 extern const PhysicalType ROW_TYPE;
 
-string TypeIdToString(PhysicalType type);
+DUCKDB_API string TypeIdToString(PhysicalType type);
 idx_t GetTypeIdSize(PhysicalType type);
 bool TypeIsConstantSize(PhysicalType type);
 bool TypeIsIntegral(PhysicalType type);
@@ -1270,7 +1270,8 @@ enum class ExceptionType {
 	INTERNAL =
 	    31, // Internal exception: exception that indicates something went wrong internally (i.e. bug in the code base)
 	INVALID_INPUT = 32, // Input or arguments error
-	OUT_OF_MEMORY = 33  // out of memory
+	OUT_OF_MEMORY = 33, // out of memory
+	PERMISSION = 34     // insufficient permissions
 };
 
 class Exception : public std::exception {
@@ -1334,6 +1335,16 @@ public:
 	}
 };
 
+class PermissionException : public StandardException {
+public:
+	DUCKDB_API explicit PermissionException(const string &msg);
+
+	template <typename... Args>
+	explicit PermissionException(const string &msg, Args... params)
+	    : PermissionException(ConstructMessage(msg, params...)) {
+	}
+};
+
 class BinderException : public StandardException {
 public:
 	DUCKDB_API explicit BinderException(const string &msg);