duckdb 0.5.2-dev748.0 → 0.5.2-dev756.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.5.2-dev748.0",
+  "version": "0.5.2-dev756.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -1815,16 +1815,19 @@ namespace duckdb {
 enum class NodeType : uint8_t { NLeaf = 0, N4 = 1, N16 = 2, N48 = 3, N256 = 4 };
 class ART;
 class Node;
+
+// Note: SwizzleablePointer assumes top 33 bits of the block_id are 0. Use a different
+// pointer implementation if that does not hold.
 class SwizzleablePointer;
+using ARTPointer = SwizzleablePointer;
 
 struct InternalType {
 	explicit InternalType(Node *n);
 
-	void Set(uint8_t *key_p, uint16_t key_size_p, SwizzleablePointer *children_p, uint16_t children_size_p);
-
+	void Set(uint8_t *key_p, uint16_t key_size_p, ARTPointer *children_p, uint16_t children_size_p);
 	uint8_t *key;
 	uint16_t key_size;
-	SwizzleablePointer *children;
+	ARTPointer *children;
 	uint16_t children_size;
 };
 
@@ -2040,29 +2043,43 @@ private:
 
 namespace duckdb {
 
+class ART;
+class Node;
+
+// SwizzleablePointer assumes that the 64-bit blockId always has 0s in the top
+// 33 bits. It thus uses 8 bytes of memory rather than 12.
 class SwizzleablePointer {
 public:
 	~SwizzleablePointer();
 	explicit SwizzleablePointer(duckdb::MetaBlockReader &reader);
 	SwizzleablePointer() : pointer(0) {};
 
-	uint64_t pointer;
+	BlockPointer Serialize(ART &art, duckdb::MetaBlockWriter &writer);
 
 	//! Transforms from Node* to uint64_t
 	SwizzleablePointer &operator=(const Node *ptr);
+
+	//! Unswizzle the pointer (if possible)
+	Node *Unswizzle(ART &art);
+
+	operator bool() const {
+		return pointer;
+	}
+
+	//! Deletes the underlying object (if necessary) and set the pointer to null_ptr
+	void Reset();
+
+private:
+	uint64_t pointer;
+
 	friend bool operator!=(const SwizzleablePointer &s_ptr, const uint64_t &ptr);
 
 	//! Extracts block info from swizzled pointer
 	BlockPointer GetSwizzledBlockInfo();
 	//! Checks if pointer is swizzled
 	bool IsSwizzled();
-	//! Deletes the underlying object (if necessary) and set the pointer to null_ptr
-	void Reset();
-	//! Unswizzle the pointer (if possible)
-	Node *Unswizzle(ART &art);
-
-	BlockPointer Serialize(ART &art, duckdb::MetaBlockWriter &writer);
 };
+
 } // namespace duckdb
 
 
@@ -2072,7 +2089,7 @@ class Node16 : public Node {
 public:
 	explicit Node16();
 	uint8_t key[16];
-	SwizzleablePointer children[16];
+	ARTPointer children[16];
 
 public:
 	//! Get position of a specific byte, returns DConstants::INVALID_INDEX if not exists
@@ -2117,7 +2134,7 @@ namespace duckdb {
 class Node256 : public Node {
 public:
 	explicit Node256();
-	SwizzleablePointer children[256];
+	ARTPointer children[256];
 
 public:
 	//! Get position of a specific byte, returns DConstants::INVALID_INDEX if not exists
@@ -2163,7 +2180,8 @@ class Node4 : public Node {
 public:
 	Node4();
 	uint8_t key[4];
-	SwizzleablePointer children[4];
+	// Pointers to the child nodes
+	ARTPointer children[4];
 
 public:
 	//! Get position of a byte, returns DConstants::INVALID_INDEX if not exists
@@ -2209,7 +2227,7 @@ class Node48 : public Node {
 public:
 	explicit Node48();
 	uint8_t child_index[256];
-	SwizzleablePointer children[48];
+	ARTPointer children[48];
 
 public:
 	//! Get position of a specific byte, returns DConstants::INVALID_INDEX if not exists
@@ -59427,7 +59445,7 @@ void Iterator::FindMinimum(Node &node) {
 	}
 	case NodeType::N256: {
 		auto &n256 = (Node256 &)node;
-		while (!n256.children[pos].pointer) {
+		while (!n256.children[pos]) {
 			pos++;
 		}
 		cur_key.Push(pos);
@@ -59788,7 +59806,7 @@ InternalType::InternalType(Node *n) {
 	}
 }
 
-void InternalType::Set(uint8_t *key_p, uint16_t key_size_p, SwizzleablePointer *children_p, uint16_t children_size_p) {
+void InternalType::Set(uint8_t *key_p, uint16_t key_size_p, ARTPointer *children_p, uint16_t children_size_p) {
 	key = key_p;
 	key_size = key_size_p;
 	children = children_p;
@@ -59969,7 +59987,7 @@ void Node::DeserializeInternal(duckdb::MetaBlockReader &reader) {
 	}
 	// Get Child offsets
 	for (idx_t i = 0; i < internal_type.children_size; i++) {
-		internal_type.children[i] = SwizzleablePointer(reader);
+		internal_type.children[i] = ARTPointer(reader);
 	}
 }
 
@@ -60217,7 +60235,7 @@ void Node16::InsertChild(Node *&node, uint8_t key_byte, Node *new_child) {
 		while (pos < node->count && n->key[pos] < key_byte) {
 			pos++;
 		}
-		if (n->children[pos] != 0) {
+		if (n->children[pos]) {
 			for (idx_t i = n->count; i > pos; i--) {
 				n->key[i] = n->key[i - 1];
 				n->children[i] = n->children[i - 1];
@@ -60255,7 +60273,7 @@ void Node16::EraseChild(Node *&node, int pos, ART &art) {
 	}
 	// set any remaining nodes as nullptr
 	for (; pos < 16; pos++) {
-		if (!n->children[pos].pointer) {
+		if (!n->children[pos]) {
 			break;
 		}
 		n->children[pos] = nullptr;
@@ -60304,7 +60322,7 @@ Node256::Node256() : Node(NodeType::N256) {
 }
 
 idx_t Node256::GetChildPos(uint8_t k) {
-	if (children[k].pointer) {
+	if (children[k]) {
 		return k;
 	} else {
 		return DConstants::INVALID_INDEX;
@@ -60313,7 +60331,7 @@ idx_t Node256::GetChildPos(uint8_t k) {
 
 idx_t Node256::GetChildGreaterEqual(uint8_t k, bool &equal) {
 	for (idx_t pos = k; pos < 256; pos++) {
-		if (children[pos].pointer) {
+		if (children[pos]) {
 			if (pos == k) {
 				equal = true;
 			} else {
@@ -60327,7 +60345,7 @@ idx_t Node256::GetChildGreaterEqual(uint8_t k, bool &equal) {
 
 idx_t Node256::GetMin() {
 	for (idx_t i = 0; i < 256; i++) {
-		if (children[i].pointer) {
+		if (children[i]) {
 			return i;
 		}
 	}
@@ -60336,7 +60354,7 @@ idx_t Node256::GetMin() {
 
 idx_t Node256::GetNextPos(idx_t pos) {
 	for (pos == DConstants::INVALID_INDEX ? pos = 0 : pos++; pos < 256; pos++) {
-		if (children[pos].pointer) {
+		if (children[pos]) {
 			return pos;
 		}
 	}
@@ -60366,7 +60384,7 @@ void Node256::EraseChild(Node *&node, int pos, ART &art) {
 		auto new_node = new Node48();
 		new_node->prefix = move(n->prefix);
 		for (idx_t i = 0; i < 256; i++) {
-			if (n->children[i].pointer) {
+			if (n->children[i]) {
 				new_node->child_index[i] = new_node->count;
 				new_node->children[new_node->count] = n->children[i];
 				n->children[i] = nullptr;
@@ -60464,7 +60482,7 @@ void Node4::InsertChild(Node *&node, uint8_t key_byte, Node *new_child) {
 		while ((pos < node->count) && (n->key[pos] < key_byte)) {
 			pos++;
 		}
-		if (n->children[pos] != 0) {
+		if (n->children[pos]) {
 			for (idx_t i = n->count; i > pos; i--) {
 				n->key[i] = n->key[i - 1];
 				n->children[i] = n->children[i - 1];
@@ -60605,10 +60623,10 @@ void Node48::InsertChild(Node *&node, uint8_t key_byte, Node *new_child) {
 	if (node->count < 48) {
 		// Insert element
 		idx_t pos = n->count;
-		if (n->children[pos].pointer) {
+		if (n->children[pos]) {
 			// find an empty position in the node list if the current position is occupied
 			pos = 0;
-			while (n->children[pos].pointer) {
+			while (n->children[pos]) {
 				pos++;
 			}
 		}
@@ -60816,15 +60834,23 @@ SwizzleablePointer::SwizzleablePointer(duckdb::MetaBlockReader &reader) {
 	}
 	idx_t pointer_size = sizeof(pointer) * 8;
 	pointer = block_id;
+	// This assumes high 32 bits of pointer are zero.
 	pointer = pointer << (pointer_size / 2);
+	D_ASSERT((pointer >> (pointer_size / 2)) == block_id);
 	pointer += offset;
 	// Set the left most bit to indicate this is a swizzled pointer and send it back to the mother-ship
 	uint64_t mask = 1;
 	mask = mask << (pointer_size - 1);
+	// This assumes the 33rd most significant bit of the block_id is zero.
 	pointer |= mask;
 }
 
 SwizzleablePointer &SwizzleablePointer::operator=(const Node *ptr) {
+	// If the object already has a non-swizzled pointer, this will leak memory.
+	//
+	// TODO: If enabled, this assert will fire, indicating a possible leak. If an exception
+	// is thrown here, it will cause a double-free. There is some work to do to make all this safer.
+	// D_ASSERT(empty() || IsSwizzled());
 	if (sizeof(ptr) == 4) {
 		pointer = (uint32_t)(size_t)ptr;
 	} else {
@@ -60840,11 +60866,14 @@ bool operator!=(const SwizzleablePointer &s_ptr, const uint64_t &ptr) {
 BlockPointer SwizzleablePointer::GetSwizzledBlockInfo() {
 	D_ASSERT(IsSwizzled());
 	idx_t pointer_size = sizeof(pointer) * 8;
+	// This is destructive. Pointer will be invalid after this operation.
+	// That's okay because this is only ever called from Unswizzle.
 	pointer = pointer & ~(1ULL << (pointer_size - 1));
 	uint32_t block_id = pointer >> (pointer_size / 2);
 	uint32_t offset = pointer & 0xffffffff;
 	return {block_id, offset};
 }
+
 bool SwizzleablePointer::IsSwizzled() {
 	idx_t pointer_size = sizeof(pointer) * 8;
 	return (pointer >> (pointer_size - 1)) & 1;
@@ -60877,6 +60906,7 @@ BlockPointer SwizzleablePointer::Serialize(ART &art, duckdb::MetaBlockWriter &wr
 		return {(block_id_t)DConstants::INVALID_INDEX, (uint32_t)DConstants::INVALID_INDEX};
 	}
 }
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -101058,9 +101088,16 @@ void DatePartFun::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+
 namespace duckdb {
 
 struct DateSub {
+	static int64_t SubtractMicros(timestamp_t startdate, timestamp_t enddate) {
+		const auto start = Timestamp::GetEpochMicroSeconds(startdate);
+		const auto end = Timestamp::GetEpochMicroSeconds(enddate);
+		return SubtractOperatorOverflowCheck::Operation<int64_t, int64_t, int64_t>(end, start);
+	}
+
 	template <class TA, class TB, class TR, class OP>
 	static inline void BinaryExecute(Vector &left, Vector &right, Vector &result, idx_t count) {
 		BinaryExecutor::ExecuteWithNulls<TA, TB, TR>(
@@ -101147,55 +101184,49 @@ struct DateSub {
 	struct DayOperator {
 		template <class TA, class TB, class TR>
 		static inline TR Operation(TA startdate, TB enddate) {
-			return (Timestamp::GetEpochMicroSeconds(enddate) - Timestamp::GetEpochMicroSeconds(startdate)) /
-			       Interval::MICROS_PER_DAY;
+			return SubtractMicros(startdate, enddate) / Interval::MICROS_PER_DAY;
 		}
 	};
 
 	struct WeekOperator {
 		template <class TA, class TB, class TR>
 		static inline TR Operation(TA startdate, TB enddate) {
-			return (Timestamp::GetEpochMicroSeconds(enddate) - Timestamp::GetEpochMicroSeconds(startdate)) /
-			       Interval::MICROS_PER_WEEK;
+			return SubtractMicros(startdate, enddate) / Interval::MICROS_PER_WEEK;
 		}
 	};
 
 	struct MicrosecondsOperator {
 		template <class TA, class TB, class TR>
 		static inline TR Operation(TA startdate, TB enddate) {
-			return (Timestamp::GetEpochMicroSeconds(enddate) - Timestamp::GetEpochMicroSeconds(startdate));
+			return SubtractMicros(startdate, enddate);
 		}
 	};
 
 	struct MillisecondsOperator {
 		template <class TA, class TB, class TR>
 		static inline TR Operation(TA startdate, TB enddate) {
-			return (Timestamp::GetEpochMicroSeconds(enddate) - Timestamp::GetEpochMicroSeconds(startdate)) /
-			       Interval::MICROS_PER_MSEC;
+			return SubtractMicros(startdate, enddate) / Interval::MICROS_PER_MSEC;
 		}
 	};
 
 	struct SecondsOperator {
 		template <class TA, class TB, class TR>
 		static inline TR Operation(TA startdate, TB enddate) {
-			return (Timestamp::GetEpochMicroSeconds(enddate) - Timestamp::GetEpochMicroSeconds(startdate)) /
-			       Interval::MICROS_PER_SEC;
+			return SubtractMicros(startdate, enddate) / Interval::MICROS_PER_SEC;
 		}
 	};
 
 	struct MinutesOperator {
 		template <class TA, class TB, class TR>
 		static inline TR Operation(TA startdate, TB enddate) {
-			return (Timestamp::GetEpochMicroSeconds(enddate) - Timestamp::GetEpochMicroSeconds(startdate)) /
-			       Interval::MICROS_PER_MINUTE;
+			return SubtractMicros(startdate, enddate) / Interval::MICROS_PER_MINUTE;
 		}
 	};
 
 	struct HoursOperator {
 		template <class TA, class TB, class TR>
 		static inline TR Operation(TA startdate, TB enddate) {
-			return (Timestamp::GetEpochMicroSeconds(enddate) - Timestamp::GetEpochMicroSeconds(startdate)) /
-			       Interval::MICROS_PER_HOUR;
+			return SubtractMicros(startdate, enddate) / Interval::MICROS_PER_HOUR;
 		}
 	};
 };