duckdb 1.1.2-dev2.0 → 1.1.2-dev6.0

package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp

@@ -19,6 +19,7 @@
 #include "duckdb/parser/parsed_data/pragma_info.hpp"
 #include "duckdb/parser/parsed_data/transaction_info.hpp"
 #include "duckdb/parser/parsed_data/vacuum_info.hpp"
+#include "duckdb/parser/parsed_data/exported_table_data.hpp"
 
 namespace duckdb {
 
@@ -36,6 +37,9 @@ unique_ptr<ParseInfo> ParseInfo::Deserialize(Deserializer &deserializer) {
 	case ParseInfoType::ATTACH_INFO:
 		result = AttachInfo::Deserialize(deserializer);
 		break;
+	case ParseInfoType::BOUND_EXPORT_DATA:
+		result = BoundExportData::Deserialize(deserializer);
+		break;
 	case ParseInfoType::COPY_DATABASE_INFO:
 		result = CopyDatabaseInfo::Deserialize(deserializer);
 		break;
@@ -225,6 +229,17 @@ unique_ptr<ParseInfo> AttachInfo::Deserialize(Deserializer &deserializer) {
 	return std::move(result);
 }
 
+void BoundExportData::Serialize(Serializer &serializer) const {
+	ParseInfo::Serialize(serializer);
+	serializer.WritePropertyWithDefault<vector<ExportedTableInfo>>(200, "data", data);
+}
+
+unique_ptr<ParseInfo> BoundExportData::Deserialize(Deserializer &deserializer) {
+	auto result = duckdb::unique_ptr<BoundExportData>(new BoundExportData());
+	deserializer.ReadPropertyWithDefault<vector<ExportedTableInfo>>(200, "data", result->data);
+	return std::move(result);
+}
+
 void ChangeColumnTypeInfo::Serialize(Serializer &serializer) const {
 	AlterTableInfo::Serialize(serializer);
 	serializer.WritePropertyWithDefault<string>(400, "column_name", column_name);

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -376,8 +376,9 @@ void SingleFileBlockManager::MarkBlockAsUsed(block_id_t block_id) {
 		}
 		max_block++;
 	} else if (free_list.find(block_id) != free_list.end()) {
-		// block is currently int he free list - erase
+		// block is currently in the free list - erase
 		free_list.erase(block_id);
+		newly_freed_list.erase(block_id);
 	} else {
 		// block is already in use - increase reference count
 		IncreaseBlockReferenceCountInternal(block_id);

package/src/duckdb/src/storage/statistics/distinct_statistics.cpp

@@ -7,14 +7,11 @@
 
 namespace duckdb {
 
-DistinctStatistics::DistinctStatistics()
-    : log(make_uniq<HyperLogLog>()), sample_count(0), total_count(0),
-      hash_vec(LogicalType::HASH, STANDARD_VECTOR_SIZE) {
+DistinctStatistics::DistinctStatistics() : log(make_uniq<HyperLogLog>()), sample_count(0), total_count(0) {
 }
 
 DistinctStatistics::DistinctStatistics(unique_ptr<HyperLogLog> log, idx_t sample_count, idx_t total_count)
-    : log(std::move(log)), sample_count(sample_count), total_count(total_count),
-      hash_vec(LogicalType::HASH, STANDARD_VECTOR_SIZE) {
+    : log(std::move(log)), sample_count(sample_count), total_count(total_count) {
 }
 
 unique_ptr<DistinctStatistics> DistinctStatistics::Copy() const {
@@ -41,6 +38,7 @@ void DistinctStatistics::Update(Vector &v, idx_t count, bool sample) {
 	sample_count += count;
 
 	lock_guard<mutex> guard(lock);
+	Vector hash_vec(LogicalType::HASH, count);
 	VectorOperations::Hash(v, hash_vec, count);
 
 	UnifiedVectorFormat vdata;

package/src/duckdb/src/storage/storage_info.cpp

@@ -28,13 +28,13 @@ static const StorageVersionInfo storage_version_info[] = {
     {"v0.3.0", 25},  {"v0.3.1", 27}, {"v0.3.2", 31}, {"v0.3.3", 33}, {"v0.3.4", 33},  {"v0.3.5", 33},  {"v0.4.0", 33},
     {"v0.5.0", 38},  {"v0.5.1", 38}, {"v0.6.0", 39}, {"v0.6.1", 39}, {"v0.7.0", 43},  {"v0.7.1", 43},  {"v0.8.0", 51},
     {"v0.8.1", 51},  {"v0.9.0", 64}, {"v0.9.1", 64}, {"v0.9.2", 64}, {"v0.10.0", 64}, {"v0.10.1", 64}, {"v0.10.2", 64},
-    {"v0.10.3", 64}, {"v1.0.0", 64}, {"v1.1.0", 64}, {nullptr, 0}};
+    {"v0.10.3", 64}, {"v1.0.0", 64}, {"v1.1.0", 64}, {"v1.1.1", 64}, {"v1.1.2", 64},  {"v1.1.3", 64},  {nullptr, 0}};
 // END OF STORAGE VERSION INFO
 
 // START OF SERIALIZATION VERSION INFO
-static const SerializationVersionInfo serialization_version_info[] = {{"v0.10.0", 1}, {"v0.10.1", 1}, {"v0.10.2", 1},
-                                                                      {"v0.10.3", 2}, {"v1.0.0", 2},  {"v1.1.0", 3},
-                                                                      {"latest", 3},  {nullptr, 0}};
+static const SerializationVersionInfo serialization_version_info[] = {
+    {"v0.10.0", 1}, {"v0.10.1", 1}, {"v0.10.2", 1}, {"v0.10.3", 2}, {"v1.0.0", 2}, {"v1.1.0", 3},
+    {"v1.1.1", 3},  {"v1.1.2", 3},  {"v1.1.3", 3},  {"latest", 3},  {nullptr, 0}};
 // END OF SERIALIZATION VERSION INFO
 
 optional_idx GetStorageVersion(const char *version_string) {

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -962,8 +962,8 @@ unique_ptr<CheckpointTask> RowGroupCollection::GetCheckpointTask(CollectionCheck
 }
 
 void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &global_stats) {
-	auto segments = row_groups->MoveSegments();
 	auto l = row_groups->Lock();
+	auto segments = row_groups->MoveSegments(l);
 
 	CollectionCheckpointState checkpoint_state(*this, writer, segments, global_stats);
 

package/src/duckdb/src/storage/table/row_version_manager.cpp

@@ -139,11 +139,15 @@ void RowVersionManager::CleanupAppend(transaction_t lowest_active_transaction, i
 	for (idx_t vector_idx = start_vector_idx; vector_idx <= end_vector_idx; vector_idx++) {
 		idx_t vcount =
 		    vector_idx == end_vector_idx ? row_group_end - end_vector_idx * STANDARD_VECTOR_SIZE : STANDARD_VECTOR_SIZE;
-		auto &info = *vector_info[vector_idx];
 		if (vcount != STANDARD_VECTOR_SIZE) {
 			// not written fully - skip
 			continue;
 		}
+		if (!vector_info[vector_idx]) {
+			// already vacuumed - skip
+			continue;
+		}
+		auto &info = *vector_info[vector_idx];
 		// if we wrote the entire chunk info try to compress it
 		unique_ptr<ChunkInfo> new_info;
 		auto cleanup = info.Cleanup(lowest_active_transaction, new_info);

package/src/duckdb/src/storage/temporary_file_manager.cpp

@@ -72,7 +72,7 @@ void BlockIndexManager::SetMaxIndex(idx_t new_index) {
 			auto difference = new_index - old;
 			auto size_on_disk = difference * TEMP_FILE_BLOCK_SIZE;
 			manager->IncreaseSizeOnDisk(size_on_disk);
-			// Increase can throw, so this is only updated after it was succesfully updated
+			// Increase can throw, so this is only updated after it was successfully updated
 			max_index = new_index;
 		}
 	}

package/src/duckdb/src/transaction/duck_transaction.cpp

@@ -271,23 +271,24 @@ unique_ptr<StorageLockKey> DuckTransaction::TryGetCheckpointLock() {
 }
 
 shared_ptr<CheckpointLock> DuckTransaction::SharedLockTable(DataTableInfo &info) {
-	lock_guard<mutex> l(active_locks_lock);
+	unique_lock<mutex> transaction_lock(active_locks_lock);
 	auto entry = active_locks.find(info);
-	if (entry != active_locks.end()) {
-		// found an existing lock
-		auto lock_weak_ptr = entry->second;
-		// check if it is expired
-		auto lock = lock_weak_ptr.lock();
-		if (lock) {
-			// not expired - return it
-			return lock;
-		}
+	if (entry == active_locks.end()) {
+		entry = active_locks.insert(entry, make_pair(std::ref(info), make_uniq<ActiveTableLock>()));
+	}
+	auto &active_table_lock = *entry->second;
+	transaction_lock.unlock(); // release transaction-level lock before acquiring table-level lock
+	lock_guard<mutex> table_lock(active_table_lock.checkpoint_lock_mutex);
+	auto checkpoint_lock = active_table_lock.checkpoint_lock.lock();
+	// check if it is expired (or has never been acquired yet)
+	if (checkpoint_lock) {
+		// not expired - return it
+		return checkpoint_lock;
 	}
 	// no existing lock - obtain it
-	auto table_lock = info.GetSharedLock();
-	auto checkpoint_lock = make_shared_ptr<CheckpointLock>(std::move(table_lock));
-	// insert it into the active locks and return it
-	active_locks.insert(make_pair(std::ref(info), checkpoint_lock));
+	checkpoint_lock = make_shared_ptr<CheckpointLock>(info.GetSharedLock());
+	// store it for future reference
+	active_table_lock.checkpoint_lock = checkpoint_lock;
 	return checkpoint_lock;
 }
 

package/src/duckdb/third_party/brotli/common/brotli_platform.h

@@ -540,4 +540,4 @@ T __brotli_swap_tmp = (A)[(I)]; \
 #endif
 	}
 }
-#endif  /* BROTLI_COMMON_PLATFORM_H_ */
+#endif  /* BROTLI_COMMON_PLATFORM_H_ */

package/src/duckdb/third_party/brotli/dec/decode.cpp

@@ -2755,4 +2755,4 @@ void InverseMoveToFrontTransformForTest(uint8_t *v, brotli_reg_t l, BrotliDecode
 }
 #endif
 
-}
+}

package/src/duckdb/third_party/brotli/enc/memory.cpp

@@ -64,7 +64,7 @@ void duckdb_brotli::BrotliWipeOutMemoryManager(MemoryManager* m) {
 
 #else  /* BROTLI_ENCODER_EXIT_ON_OOM */
 
-static void SortPointers(void** items, const size_t n) {
+void SortPointers(void** items, const size_t n) {
   /* Shell sort. */
   /* TODO(eustas): fine-tune for "many slots" case */
   static const size_t gaps[] = {23, 10, 4, 1};
@@ -135,7 +135,7 @@ static void CollectGarbagePointers(MemoryManager* m) {
   }
 }
 
-void* BrotliAllocate(MemoryManager* m, size_t n) {
+void* duckdb_brotli::BrotliAllocate(MemoryManager* m, size_t n) {
   void* result = m->alloc_func(m->opaque, n);
   if (!result) {
     m->is_oom = BROTLI_TRUE;
@@ -146,14 +146,14 @@ void* BrotliAllocate(MemoryManager* m, size_t n) {
   return result;
 }
 
-void BrotliFree(MemoryManager* m, void* p) {
+void duckdb_brotli::BrotliFree(MemoryManager* m, void* p) {
   if (!p) return;
   m->free_func(m->opaque, p);
   if (m->new_freed == MAX_NEW_FREED) CollectGarbagePointers(m);
   m->pointers[NEW_FREED_OFFSET + (m->new_freed++)] = p;
 }
 
-void BrotliWipeOutMemoryManager(MemoryManager* m) {
+void duckdb_brotli::BrotliWipeOutMemoryManager(MemoryManager* m) {
   size_t i;
   CollectGarbagePointers(m);
   /* Now all unfreed pointers are in perm-allocated list. */

package/src/duckdb/third_party/fsst/libfsst.cpp

@@ -503,4 +503,4 @@ extern "C" duckdb_fsst_decoder_t duckdb_fsst_decoder(duckdb_fsst_encoder_t *enco
 	u32 cnt2 = duckdb_fsst_import(&decoder, buf);
 	assert(cnt1 == cnt2); (void) cnt1; (void) cnt2;
 	return decoder;
-}
+}

package/src/duckdb/third_party/hyperloglog/sds.cpp

@@ -1112,4 +1112,4 @@ void *sdmalloc(size_t size) { return malloc(size); }
 void *sdrealloc(void *ptr, size_t size) { return realloc(ptr,size); }
 void sdfree(void *ptr) { free(ptr); }
 
-}
+}

package/src/duckdb/third_party/hyperloglog/sds.hpp

@@ -281,4 +281,4 @@ int sdsTest(int argc, char *argv[]);
 }
 
 
-#endif
+#endif

package/src/duckdb/third_party/libpg_query/include/common/keywords.hpp

@@ -31,4 +31,4 @@ typedef struct PGScanKeyword {
 } PGScanKeyword;
 
 const PGScanKeyword *ScanKeywordLookup(const char *text, const PGScanKeyword *keywords, int num_keywords);
-}
+}

package/src/duckdb/third_party/libpg_query/include/datatype/timestamp.hpp

@@ -48,4 +48,4 @@ typedef struct {
 	int32_t day;       /* days, after time for alignment */
 	int32_t month;     /* months and years, after time for alignment */
 } PGInterval;
-}
+}

package/src/duckdb/third_party/libpg_query/include/mb/pg_wchar.hpp

@@ -25,4 +25,4 @@
  */
 namespace duckdb_libpgquery {
 typedef unsigned int pg_wchar;
-}
+}

package/src/duckdb/third_party/libpg_query/include/nodes/bitmapset.hpp

@@ -99,4 +99,4 @@ int bms_next_member(const PGBitmapset *a, int prevbit);
 /* support for hashtables using Bitmapsets as keys: */
 uint32_t bms_hash_value(const PGBitmapset *a);
 
-}
+}

package/src/duckdb/third_party/libpg_query/include/nodes/lockoptions.hpp

@@ -41,4 +41,4 @@ typedef enum PGLockWaitPolicy {
 	LockWaitError
 } PGLockWaitPolicy;
 
-}
+}

package/src/duckdb/third_party/libpg_query/include/nodes/makefuncs.hpp

@@ -66,4 +66,4 @@ PGDefElem *makeDefElemExtended(const char *nameSpace, const char *name, PGNode *
 
 PGGroupingSet *makeGroupingSet(GroupingSetKind kind, PGList *content, int location);
 
-}
+}

package/src/duckdb/third_party/libpg_query/include/nodes/pg_list.hpp

@@ -336,4 +336,4 @@ PGList *list_copy_tail(const PGList *list, int nskip);
 int	length(PGList *list);
 #endif							/* ENABLE_LIST_COMPAT */
 
-}
+}

package/src/duckdb/third_party/libpg_query/include/nodes/value.hpp

@@ -57,4 +57,4 @@ PGValue *makeFloat(char *numericStr);
 PGValue *makeString(const char *str);
 PGValue *makeBitString(char *str);
 
-}
+}

package/src/duckdb/third_party/libpg_query/include/parser/gramparse.hpp

@@ -65,4 +65,4 @@ int base_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner);
 void parser_init(base_yy_extra_type *yyext);
 int base_yyparse(core_yyscan_t yyscanner);
 
-}
+}

package/src/duckdb/third_party/libpg_query/include/parser/parser.hpp

@@ -37,4 +37,4 @@ std::vector<PGSimplifiedToken> tokenize(const char *str);
 PGList *SystemFuncName(const char *name);
 PGTypeName *SystemTypeName(const char *name);
 
-}
+}

package/src/duckdb/third_party/libpg_query/include/parser/scanner.hpp

@@ -124,4 +124,4 @@ int core_yylex(core_YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner);
 int scanner_errposition(int location, core_yyscan_t yyscanner);
 void scanner_yyerror(const char *message, core_yyscan_t yyscanner);
 
-}
+}

package/src/duckdb/third_party/libpg_query/include/parser/scansup.hpp

@@ -27,4 +27,4 @@ bool scanner_isspace(char ch);
 void set_preserve_identifier_case(bool downcase);
 bool get_preserve_identifier_case();
 
-}
+}

package/src/duckdb/third_party/libpg_query/include/pg_functions.hpp

@@ -60,4 +60,4 @@ PGDefElem *defWithOids(bool value);
 typedef unsigned int pg_wchar;
 unsigned char *unicode_to_utf8(pg_wchar c, unsigned char *utf8string);
 
-}
+}

package/src/duckdb/third_party/libpg_query/pg_functions.cpp

@@ -274,4 +274,4 @@ PGNode *newNode(size_t size, PGNodeTag type) {
 	result->type = type;
 	return result;
 }
-}
+}

package/src/duckdb/third_party/libpg_query/src_backend_nodes_list.cpp

@@ -537,4 +537,4 @@ list_copy_tail(const PGList *oldlist, int nskip)
 int			length(const PGList *list);
 
 
-}
+}

package/src/duckdb/third_party/libpg_query/src_backend_nodes_makefuncs.cpp

@@ -302,4 +302,4 @@ PGGroupingSet *makeGroupingSet(GroupingSetKind kind, PGList *content, int locati
 	n->location = location;
 	return n;
 }
-}
+}

package/src/duckdb/third_party/libpg_query/src_backend_nodes_value.cpp

@@ -71,4 +71,4 @@ PGValue *makeString(const char *str) {
  * Caller is responsible for passing a palloc'd string.
  */
 
-}
+}