duckdb 0.4.1-dev108.0 → 0.4.1-dev129.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.4.1-dev108.0",
+  "version": "0.4.1-dev129.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -176123,6 +176123,7 @@ private:
 
 
 
+
 namespace duckdb {
 
 WriteOverflowStringsToDisk::WriteOverflowStringsToDisk(DatabaseInstance &db)
@@ -176143,19 +176144,29 @@ void WriteOverflowStringsToDisk::WriteString(string_t string, block_id_t &result
 		handle = buffer_manager.Allocate(Storage::BLOCK_SIZE);
 	}
 	// first write the length of the string
-	if (block_id == INVALID_BLOCK || offset + sizeof(uint32_t) >= STRING_SPACE) {
+	if (block_id == INVALID_BLOCK || offset + 2 * sizeof(uint32_t) >= STRING_SPACE) {
 		AllocateNewBlock(block_manager.GetFreeBlockId());
 	}
 	result_block = block_id;
 	result_offset = offset;
 
-	// write the length field
-	auto string_length = string.GetSize();
-	Store<uint32_t>(string_length, handle->node->buffer + offset);
-	offset += sizeof(uint32_t);
+	// GZIP the string
+	auto uncompressed_size = string.GetSize();
+	MiniZStream s;
+	size_t compressed_size = 0;
+	compressed_size = s.MaxCompressedLength(uncompressed_size);
+	auto compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
+	s.Compress((const char *)string.GetDataUnsafe(), uncompressed_size, (char *)compressed_buf.get(), &compressed_size);
+	string_t compressed_string((const char *)compressed_buf.get(), compressed_size);
+
+	// store sizes
+	Store<uint32_t>(compressed_size, handle->node->buffer + offset);
+	Store<uint32_t>(uncompressed_size, handle->node->buffer + offset + sizeof(uint32_t));
+
 	// now write the remainder of the string
-	auto strptr = string.GetDataUnsafe();
-	uint32_t remaining = string_length;
+	offset += 2 * sizeof(uint32_t);
+	auto strptr = compressed_string.GetDataUnsafe();
+	uint32_t remaining = compressed_size;
 	while (remaining > 0) {
 		uint32_t to_write = MinValue<uint32_t>(remaining, STRING_SPACE - offset);
 		if (to_write > 0) {
@@ -178838,7 +178849,7 @@ public:
 	//! Base size of big string marker (block id + offset)
 	static constexpr idx_t BIG_STRING_MARKER_BASE_SIZE = sizeof(block_id_t) + sizeof(int32_t);
 	//! The marker size of the big string
-	static constexpr idx_t BIG_STRING_MARKER_SIZE = BIG_STRING_MARKER_BASE_SIZE + sizeof(uint16_t);
+	static constexpr idx_t BIG_STRING_MARKER_SIZE = BIG_STRING_MARKER_BASE_SIZE;
 	//! The size below which the segment is compacted on flushing
 	static constexpr size_t COMPACTION_FLUSH_LIMIT = (size_t)Storage::BLOCK_SIZE / 5 * 4;
 
@@ -178877,11 +178888,16 @@ public:
 				return i;
 			}
 			remaining_space -= sizeof(int32_t);
+			auto dictionary = GetDictionary(segment, *handle);
 			if (!data.validity.RowIsValid(source_idx)) {
-				// null value is stored as -1
-				result_data[target_idx] = 0;
+				// null value is stored as a copy of the last value, this is done to be able to efficiently do the
+				// string_length calculation
+				if (target_idx > 0) {
+					result_data[target_idx] = result_data[target_idx - 1];
+				} else {
+					result_data[target_idx] = 0;
+				}
 			} else {
-				auto dictionary = GetDictionary(segment, *handle);
 				auto end = handle->node->buffer + dictionary.end;
 
 				dictionary.Verify();
@@ -178905,7 +178921,7 @@ public:
 					// Unknown string, continue
 					// non-null value, check if we can fit it within the block
 					idx_t string_length = source_data[source_idx].GetSize();
-					idx_t dictionary_length = string_length + sizeof(uint16_t);
+					idx_t dictionary_length = string_length;
 
 					// determine whether or not we have space in the block for this string
 					bool use_overflow_block = false;
@@ -178938,16 +178954,17 @@ public:
 						// string fits in block, append to dictionary and increment dictionary position
 						D_ASSERT(string_length < NumericLimits<uint16_t>::Maximum());
 						dictionary.size += required_space;
-						auto dict_pos = end - dictionary.size; // first write the length as u16
-						Store<uint16_t>(string_length, dict_pos);
+						auto dict_pos = end - dictionary.size;
 						// now write the actual string data into the dictionary
-						memcpy(dict_pos + sizeof(uint16_t), source_data[source_idx].GetDataUnsafe(), string_length);
+						memcpy(dict_pos, source_data[source_idx].GetDataUnsafe(), string_length);
 					}
 					D_ASSERT(RemainingSpace(segment, *handle) <= Storage::BLOCK_SIZE);
 					// place the dictionary offset into the set of vectors
 					dictionary.Verify();
 
-					result_data[target_idx] = dictionary.size;
+					// note: for overflow strings we write negative value
+					result_data[target_idx] = use_overflow_block ? -1 * dictionary.size : dictionary.size;
+
 					if (DUPLICATE_ELIMINATE) {
 						seen_strings->insert({source_data[source_idx].GetString(), dictionary.size});
 					}
@@ -178973,17 +178990,18 @@ public:
 	static void WriteString(ColumnSegment &segment, string_t string, block_id_t &result_block, int32_t &result_offset);
 	static void WriteStringMemory(ColumnSegment &segment, string_t string, block_id_t &result_block,
 	                              int32_t &result_offset);
-	static string_t ReadString(ColumnSegment &segment, Vector &result, block_id_t block, int32_t offset);
-	static string_t ReadString(data_ptr_t target, int32_t offset);
+	static string_t ReadOverflowString(ColumnSegment &segment, Vector &result, block_id_t block, int32_t offset);
+	static string_t ReadString(data_ptr_t target, int32_t offset, uint32_t string_length);
+	static string_t ReadStringWithLength(data_ptr_t target, int32_t offset);
 	static void WriteStringMarker(data_ptr_t target, block_id_t block_id, int32_t offset);
 	static void ReadStringMarker(data_ptr_t target, block_id_t &block_id, int32_t &offset);
 
 	static string_location_t FetchStringLocation(StringDictionaryContainer dict, data_ptr_t baseptr,
 	                                             int32_t dict_offset);
 	static string_t FetchStringFromDict(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
-	                                    data_ptr_t baseptr, int32_t dict_offset);
+	                                    data_ptr_t baseptr, int32_t dict_offset, uint32_t string_length);
 	static string_t FetchString(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
-	                            data_ptr_t baseptr, string_location_t location);
+	                            data_ptr_t baseptr, string_location_t location, uint32_t string_length);
 };
 } // namespace duckdb
 
@@ -180464,6 +180482,8 @@ bool RLEFun::TypeIsSupported(PhysicalType type) {
 } // namespace duckdb
 
 
+
+
 namespace duckdb {
 
 //===--------------------------------------------------------------------===//
@@ -180541,8 +180561,14 @@ void UncompressedStringStorage::StringScanPartial(ColumnSegment &segment, Column
 	auto base_data = (int32_t *)(baseptr + DICTIONARY_HEADER_SIZE);
 	auto result_data = FlatVector::GetData<string_t>(result);
 
+	int32_t previous_offset = start > 0 ? base_data[start - 1] : 0;
+
 	for (idx_t i = 0; i < scan_count; i++) {
-		result_data[result_offset + i] = FetchStringFromDict(segment, dict, result, baseptr, base_data[start + i]);
+		// std::abs used since offsets can be negative to indicate big strings
+		uint32_t string_length = std::abs(base_data[start + i]) - std::abs(previous_offset);
+		result_data[result_offset + i] =
+		    FetchStringFromDict(segment, dict, result, baseptr, base_data[start + i], string_length);
+		previous_offset = base_data[start + i];
 	}
 }
 
@@ -180577,7 +180603,15 @@ void UncompressedStringStorage::StringFetchRow(ColumnSegment &segment, ColumnFet
 	auto base_data = (int32_t *)(baseptr + DICTIONARY_HEADER_SIZE);
 	auto result_data = FlatVector::GetData<string_t>(result);
 
-	result_data[result_idx] = FetchStringFromDict(segment, dict, result, baseptr, base_data[row_id]);
+	auto dict_offset = base_data[row_id];
+	uint32_t string_length;
+	if ((idx_t)row_id == 0) {
+		// edge case where this is the first string in the dict
+		string_length = std::abs(dict_offset);
+	} else {
+		string_length = std::abs(dict_offset) - std::abs(base_data[row_id - 1]);
+	}
+	result_data[result_idx] = FetchStringFromDict(segment, dict, result, baseptr, dict_offset, string_length);
 }
 
 //===--------------------------------------------------------------------===//
@@ -180711,8 +180745,8 @@ void UncompressedStringStorage::WriteStringMemory(ColumnSegment &segment, string
 	state.head->offset += total_length;
 }
 
-string_t UncompressedStringStorage::ReadString(ColumnSegment &segment, Vector &result, block_id_t block,
-                                               int32_t offset) {
+string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, Vector &result, block_id_t block,
+                                                       int32_t offset) {
 	D_ASSERT(block != INVALID_BLOCK);
 	D_ASSERT(offset < Storage::BLOCK_SIZE);
 
@@ -180724,37 +180758,52 @@ string_t UncompressedStringStorage::ReadString(ColumnSegment &segment, Vector &r
 		auto block_handle = buffer_manager.RegisterBlock(block);
 		auto handle = buffer_manager.Pin(block_handle);
 
-		uint32_t length = Load<uint32_t>(handle->node->buffer + offset);
-		uint32_t remaining = length;
-		offset += sizeof(uint32_t);
-
-		// allocate a buffer to store the string
-		auto alloc_size = MaxValue<idx_t>(Storage::BLOCK_SIZE, length + sizeof(uint32_t));
-		auto target_handle = buffer_manager.Allocate(alloc_size);
-		auto target_ptr = target_handle->node->buffer;
-		// write the length in this block as well
-		Store<uint32_t>(length, target_ptr);
-		target_ptr += sizeof(uint32_t);
-		// now append the string to the single buffer
-		while (remaining > 0) {
-			idx_t to_write = MinValue<idx_t>(remaining, Storage::BLOCK_SIZE - sizeof(block_id_t) - offset);
-			memcpy(target_ptr, handle->node->buffer + offset, to_write);
+		// read header
+		uint32_t compressed_size = Load<uint32_t>(handle->node->buffer + offset);
+		uint32_t uncompressed_size = Load<uint32_t>(handle->node->buffer + offset + sizeof(uint32_t));
+		uint32_t remaining = compressed_size;
+		offset += 2 * sizeof(uint32_t);
 
-			remaining -= to_write;
-			offset += to_write;
-			target_ptr += to_write;
-			if (remaining > 0) {
-				// read the next block
-				block_id_t next_block = Load<block_id_t>(handle->node->buffer + offset);
-				block_handle = buffer_manager.RegisterBlock(next_block);
-				handle = buffer_manager.Pin(block_handle);
-				offset = 0;
+		data_ptr_t decompression_ptr;
+		std::unique_ptr<data_t[]> decompression_buffer;
+
+		// If string is in single block we decompress straight from it, else we copy first
+		if (remaining <= Storage::BLOCK_SIZE - sizeof(block_id_t) - offset) {
+			decompression_ptr = handle->node->buffer + offset;
+		} else {
+			decompression_buffer = std::unique_ptr<data_t[]>(new data_t[compressed_size]);
+			auto target_ptr = decompression_buffer.get();
+
+			// now append the string to the single buffer
+			while (remaining > 0) {
+				idx_t to_write = MinValue<idx_t>(remaining, Storage::BLOCK_SIZE - sizeof(block_id_t) - offset);
+				memcpy(target_ptr, handle->node->buffer + offset, to_write);
+
+				remaining -= to_write;
+				offset += to_write;
+				target_ptr += to_write;
+				if (remaining > 0) {
+					// read the next block
+					block_id_t next_block = Load<block_id_t>(handle->node->buffer + offset);
+					block_handle = buffer_manager.RegisterBlock(next_block);
+					handle = buffer_manager.Pin(block_handle);
+					offset = 0;
+				}
 			}
+			decompression_ptr = decompression_buffer.get();
 		}
 
-		auto final_buffer = target_handle->node->buffer;
-		StringVector::AddHandle(result, move(target_handle));
-		return ReadString(final_buffer, 0);
+		// overflow strings on disk are gzipped, decompress here
+		auto decompressed_target_handle =
+		    buffer_manager.Allocate(MaxValue<idx_t>(Storage::BLOCK_SIZE, uncompressed_size));
+		auto decompressed_target_ptr = decompressed_target_handle->node->buffer;
+		MiniZStream s;
+		s.Decompress((const char *)decompression_ptr, compressed_size, (char *)decompressed_target_ptr,
+		             uncompressed_size);
+
+		auto final_buffer = decompressed_target_handle->node->buffer;
+		StringVector::AddHandle(result, move(decompressed_target_handle));
+		return ReadString(final_buffer, 0, uncompressed_size);
 	} else {
 		// read the overflow string from memory
 		// first pin the handle, if it is not pinned yet
@@ -180763,11 +180812,17 @@ string_t UncompressedStringStorage::ReadString(ColumnSegment &segment, Vector &r
 		auto handle = buffer_manager.Pin(entry->second->block);
 		auto final_buffer = handle->node->buffer;
 		StringVector::AddHandle(result, move(handle));
-		return ReadString(final_buffer, offset);
+		return ReadStringWithLength(final_buffer, offset);
 	}
 }
 
-string_t UncompressedStringStorage::ReadString(data_ptr_t target, int32_t offset) {
+string_t UncompressedStringStorage::ReadString(data_ptr_t target, int32_t offset, uint32_t string_length) {
+	auto ptr = target + offset;
+	auto str_ptr = (char *)(ptr);
+	return string_t(str_ptr, string_length);
+}
+
+string_t UncompressedStringStorage::ReadStringWithLength(data_ptr_t target, int32_t offset) {
 	auto ptr = target + offset;
 	auto str_length = Load<uint32_t>(ptr);
 	auto str_ptr = (char *)(ptr + sizeof(uint32_t));
@@ -180775,16 +180830,12 @@ string_t UncompressedStringStorage::ReadString(data_ptr_t target, int32_t offset
 }
 
 void UncompressedStringStorage::WriteStringMarker(data_ptr_t target, block_id_t block_id, int32_t offset) {
-	uint16_t length = BIG_STRING_MARKER;
-	memcpy(target, &length, sizeof(uint16_t));
-	target += sizeof(uint16_t);
 	memcpy(target, &block_id, sizeof(block_id_t));
 	target += sizeof(block_id_t);
 	memcpy(target, &offset, sizeof(int32_t));
 }
 
 void UncompressedStringStorage::ReadStringMarker(data_ptr_t target, block_id_t &block_id, int32_t &offset) {
-	target += sizeof(uint16_t);
 	memcpy(&block_id, target, sizeof(block_id_t));
 	target += sizeof(block_id_t);
 	memcpy(&offset, target, sizeof(int32_t));
@@ -180792,37 +180843,31 @@ void UncompressedStringStorage::ReadStringMarker(data_ptr_t target, block_id_t &
 
 string_location_t UncompressedStringStorage::FetchStringLocation(StringDictionaryContainer dict, data_ptr_t baseptr,
                                                                  int32_t dict_offset) {
-	D_ASSERT(dict_offset >= 0 && dict_offset <= Storage::BLOCK_SIZE);
-	if (dict_offset == 0) {
-		return string_location_t(INVALID_BLOCK, 0);
-	}
-	// look up result in dictionary
-	auto dict_end = baseptr + dict.end;
-	auto dict_pos = dict_end - dict_offset;
-	auto string_length = Load<uint16_t>(dict_pos);
-	string_location_t result;
-	if (string_length == BIG_STRING_MARKER) {
-		ReadStringMarker(dict_pos, result.block_id, result.offset);
+	D_ASSERT(dict_offset >= -1 * Storage::BLOCK_SIZE && dict_offset <= Storage::BLOCK_SIZE);
+	if (dict_offset < 0) {
+		string_location_t result;
+		ReadStringMarker(baseptr + dict.end - (-1 * dict_offset), result.block_id, result.offset);
+		return result;
 	} else {
-		result.block_id = INVALID_BLOCK;
-		result.offset = dict_offset;
+		return string_location_t(INVALID_BLOCK, dict_offset);
 	}
-	return result;
 }
 
 string_t UncompressedStringStorage::FetchStringFromDict(ColumnSegment &segment, StringDictionaryContainer dict,
-                                                        Vector &result, data_ptr_t baseptr, int32_t dict_offset) {
+                                                        Vector &result, data_ptr_t baseptr, int32_t dict_offset,
+                                                        uint32_t string_length) {
 	// fetch base data
 	D_ASSERT(dict_offset <= Storage::BLOCK_SIZE);
 	string_location_t location = FetchStringLocation(dict, baseptr, dict_offset);
-	return FetchString(segment, dict, result, baseptr, location);
+	return FetchString(segment, dict, result, baseptr, location, string_length);
 }
 
 string_t UncompressedStringStorage::FetchString(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
-                                                data_ptr_t baseptr, string_location_t location) {
+                                                data_ptr_t baseptr, string_location_t location,
+                                                uint32_t string_length) {
 	if (location.block_id != INVALID_BLOCK) {
 		// big string marker: read from separate block
-		return ReadString(segment, result, location.block_id, location.offset);
+		return ReadOverflowString(segment, result, location.block_id, location.offset);
 	} else {
 		if (location.offset == 0) {
 			return string_t(nullptr, 0);
@@ -180830,9 +180875,8 @@ string_t UncompressedStringStorage::FetchString(ColumnSegment &segment, StringDi
 		// normal string: read string from this block
 		auto dict_end = baseptr + dict.end;
 		auto dict_pos = dict_end - location.offset;
-		auto string_length = Load<uint16_t>(dict_pos);
 
-		auto str_ptr = (char *)(dict_pos + sizeof(uint16_t));
+		auto str_ptr = (char *)(dict_pos);
 		return string_t(str_ptr, string_length);
 	}
 }
@@ -185085,7 +185129,7 @@ string ValidityStatistics::ToString() const {
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 33;
+const uint64_t VERSION_NUMBER = 35;
 
 } // namespace duckdb
 

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "d3b4eba6a"
-#define DUCKDB_VERSION "v0.4.1-dev108"
+#define DUCKDB_SOURCE_ID "9789bcbb0"
+#define DUCKDB_VERSION "v0.4.1-dev129"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //