duckdb 0.5.2-dev1460.0 → 0.5.2-dev1468.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.5.2-dev1460.0",
+  "version": "0.5.2-dev1468.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -80586,7 +80586,7 @@ public:
 	idx_t current_index;
 	TableAppendState current_append_state;
 	unique_ptr<RowGroupCollection> current_collection;
-	unique_ptr<OptimisticDataWriter> writer;
+	OptimisticDataWriter *writer;
 	bool written_to_disk;
 
 	void FlushToDisk() {
@@ -80636,16 +80636,17 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, GlobalSinkSt
 	PhysicalInsert::ResolveDefaults(table, chunk, column_index_map, lstate.default_executor, lstate.insert_chunk);
 
 	if (!lstate.current_collection) {
+		lock_guard<mutex> l(gstate.lock);
 		// no collection yet: create a new one
 		lstate.CreateNewCollection(table, insert_types);
-		lstate.writer = make_unique<OptimisticDataWriter>(gstate.table->storage.get());
+		lstate.writer = gstate.table->storage->CreateOptimisticWriter(context.client);
 	} else if (lstate.current_index != lstate.batch_index) {
 		// batch index has changed: move the old collection to the global state and create a new collection
 		TransactionData tdata(0, 0);
 		lstate.current_collection->FinalizeAppend(tdata, lstate.current_append_state);
 		lstate.FlushToDisk();
 
-		gstate.AddCollection(context.client, lstate.current_index, move(lstate.current_collection), lstate.writer.get(),
+		gstate.AddCollection(context.client, lstate.current_index, move(lstate.current_collection), lstate.writer,
 		                     &lstate.written_to_disk);
 		lstate.CreateNewCollection(table, insert_types);
 	}
@@ -80715,7 +80716,7 @@ SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event,
 	// now that we have created all of the mergers, perform the actual merging
 	vector<unique_ptr<RowGroupCollection>> final_collections;
 	final_collections.reserve(mergers.size());
-	auto writer = make_unique<OptimisticDataWriter>(&storage);
+	auto writer = storage.CreateOptimisticWriter(context);
 	for (auto &merger : mergers) {
 		final_collections.push_back(merger->Flush(*writer));
 	}
@@ -81415,7 +81416,7 @@ public:
 	ExpressionExecutor default_executor;
 	TableAppendState local_append_state;
 	unique_ptr<RowGroupCollection> local_collection;
-	unique_ptr<OptimisticDataWriter> writer;
+	OptimisticDataWriter *writer;
 };
 
 unique_ptr<GlobalSinkState> PhysicalInsert::GetGlobalSinkState(ClientContext &context) const {
@@ -81494,13 +81495,14 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, GlobalSinkState &
 		D_ASSERT(!return_chunk);
 		// parallel append
 		if (!lstate.local_collection) {
+			lock_guard<mutex> l(gstate.lock);
 			auto &table_info = table->storage->info;
 			auto &block_manager = TableIOManager::Get(*table->storage).GetBlockManagerForRowData();
 			lstate.local_collection =
 			    make_unique<RowGroupCollection>(table_info, block_manager, insert_types, MAX_ROW_ID);
 			lstate.local_collection->InitializeEmpty();
 			lstate.local_collection->InitializeAppend(lstate.local_append_state);
-			lstate.writer = make_unique<OptimisticDataWriter>(gstate.table->storage.get());
+			lstate.writer = gstate.table->storage->CreateOptimisticWriter(context.client);
 		}
 		table->storage->VerifyAppendConstraints(*table, context.client, lstate.insert_chunk);
 		auto new_row_group = lstate.local_collection->Append(lstate.insert_chunk, lstate.local_append_state);
@@ -87227,11 +87229,6 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalInsert &op
 
 	bool parallel_streaming_insert = !PreserveInsertionOrder(*plan);
 	bool use_batch_index = UseBatchIndex(*plan);
-	if (!op.table->storage->info->indexes.Empty()) {
-		// not for tables with indexes currently
-		parallel_streaming_insert = false;
-		use_batch_index = false;
-	}
 	if (op.return_chunk) {
 		// not supported for RETURNING (yet?)
 		parallel_streaming_insert = false;
@@ -198132,6 +198129,9 @@ public:
 	bool IsRootBlock(block_id_t root) override {
 		throw InternalException("Cannot perform IO in in-memory database!");
 	}
+	void MarkBlockAsFree(block_id_t block_id) override {
+		throw InternalException("Cannot perform IO in in-memory database!");
+	}
 	void MarkBlockAsModified(block_id_t block_id) override {
 		throw InternalException("Cannot perform IO in in-memory database!");
 	}
@@ -208337,6 +208337,11 @@ void DataTable::FinalizeLocalAppend(LocalAppendState &state) {
 	LocalStorage::FinalizeAppend(state);
 }
 
+OptimisticDataWriter *DataTable::CreateOptimisticWriter(ClientContext &context) {
+	auto &local_storage = LocalStorage::Get(context);
+	return local_storage.CreateOptimisticWriter(this);
+}
+
 void DataTable::LocalMerge(ClientContext &context, RowGroupCollection &collection) {
 	auto &local_storage = LocalStorage::Get(context);
 	local_storage.LocalMerge(this, collection);
@@ -209000,7 +209005,7 @@ void OptimisticDataWriter::Rollback() {
 	if (!written_blocks.empty()) {
 		auto &block_manager = table->info->table_io_manager->GetBlockManagerForRowData();
 		for (auto block_id : written_blocks) {
-			block_manager.MarkBlockAsModified(block_id);
+			block_manager.MarkBlockAsFree(block_id);
 		}
 	}
 }
@@ -209034,7 +209039,7 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
                                      const LogicalType &target_type, const vector<column_t> &bound_columns,
                                      Expression &cast_expr)
     : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(table, parent.optimistic_writer) {
+      optimistic_writer(table, parent.optimistic_writer), optimistic_writers(move(parent.optimistic_writers)) {
 	row_groups = parent.row_groups->AlterType(changed_idx, target_type, bound_columns, cast_expr);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -209042,7 +209047,7 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, idx_t drop_idx)
     : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(table, parent.optimistic_writer) {
+      optimistic_writer(table, parent.optimistic_writer), optimistic_writers(move(parent.optimistic_writers)) {
 	row_groups = parent.row_groups->RemoveColumn(drop_idx);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -209051,7 +209056,7 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, ColumnDefinition &new_column,
                                      Expression *default_value)
     : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(table, parent.optimistic_writer) {
+      optimistic_writer(table, parent.optimistic_writer), optimistic_writers(move(parent.optimistic_writers)) {
 	row_groups = parent.row_groups->AddColumn(new_column, default_value);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -209094,6 +209099,33 @@ void LocalTableStorage::FlushToDisk() {
 	optimistic_writer.FinalFlush();
 }
 
+bool LocalTableStorage::AppendToIndexes(Transaction &transaction, RowGroupCollection &source,
+                                        TableIndexList &index_list, const vector<LogicalType> &table_types,
+                                        row_t &start_row) {
+	// only need to scan for index append
+	// figure out which columns we need to scan for the set of indexes
+	auto columns = index_list.GetRequiredColumns();
+	// create an empty mock chunk that contains all the correct types for the table
+	DataChunk mock_chunk;
+	mock_chunk.InitializeEmpty(table_types);
+	bool success = true;
+	source.Scan(transaction, columns, [&](DataChunk &chunk) -> bool {
+		// construct the mock chunk by referencing the required columns
+		for (idx_t i = 0; i < columns.size(); i++) {
+			mock_chunk.data[columns[i]].Reference(chunk.data[i]);
+		}
+		mock_chunk.SetCardinality(chunk);
+		// append this chunk to the indexes of the table
+		if (!DataTable::AppendToIndexes(index_list, mock_chunk, start_row)) {
+			success = false;
+			return false;
+		}
+		start_row += chunk.size();
+		return true;
+	});
+	return success;
+}
+
 void LocalTableStorage::AppendToIndexes(Transaction &transaction, TableAppendState &append_state, idx_t append_count,
                                         bool append_to_table) {
 	bool constraint_violated = false;
@@ -209113,26 +209145,8 @@ void LocalTableStorage::AppendToIndexes(Transaction &transaction, TableAppendSta
 			return true;
 		});
 	} else {
-		// only need to scan for index append
-		// figure out which columns we need to scan for the set of indexes
-		auto columns = table->info->indexes.GetRequiredColumns();
-		// create an empty mock chunk that contains all the correct types for the table
-		DataChunk mock_chunk;
-		mock_chunk.InitializeEmpty(table->GetTypes());
-		row_groups->Scan(transaction, columns, [&](DataChunk &chunk) -> bool {
-			// construct the mock chunk by referencing the required columns
-			for (idx_t i = 0; i < columns.size(); i++) {
-				mock_chunk.data[columns[i]].Reference(chunk.data[i]);
-			}
-			mock_chunk.SetCardinality(chunk);
-			// append this chunk to the indexes of the table
-			if (!table->AppendToIndexes(mock_chunk, append_state.current_row)) {
-				constraint_violated = true;
-				return false;
-			}
-			append_state.current_row += chunk.size();
-			return true;
-		});
+		constraint_violated = !AppendToIndexes(transaction, *row_groups, table->info->indexes, table->GetTypes(),
+		                                       append_state.current_row);
 	}
 	if (constraint_violated) {
 		// need to revert the append
@@ -209156,8 +209170,18 @@ void LocalTableStorage::AppendToIndexes(Transaction &transaction, TableAppendSta
 	}
 }
 
+OptimisticDataWriter *LocalTableStorage::CreateOptimisticWriter() {
+	auto writer = make_unique<OptimisticDataWriter>(table);
+	optimistic_writers.push_back(move(writer));
+	return optimistic_writers.back().get();
+}
+
 void LocalTableStorage::Rollback() {
 	optimistic_writer.Rollback();
+	for (auto &writer : optimistic_writers) {
+		writer->Rollback();
+	}
+	optimistic_writers.clear();
 }
 
 //===--------------------------------------------------------------------===//
@@ -209292,9 +209316,22 @@ void LocalStorage::FinalizeAppend(LocalAppendState &state) {
 
 void LocalStorage::LocalMerge(DataTable *table, RowGroupCollection &collection) {
 	auto storage = table_manager.GetOrCreateStorage(table);
+	if (!storage->indexes.Empty()) {
+		// append data to indexes if required
+		row_t base_id = MAX_ROW_ID + storage->row_groups->GetTotalRows();
+		bool success = storage->AppendToIndexes(transaction, collection, storage->indexes, table->GetTypes(), base_id);
+		if (!success) {
+			throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
+		}
+	}
 	storage->row_groups->MergeStorage(collection);
 }
 
+OptimisticDataWriter *LocalStorage::CreateOptimisticWriter(DataTable *table) {
+	auto storage = table_manager.GetOrCreateStorage(table);
+	return storage->CreateOptimisticWriter();
+}
+
 bool LocalStorage::ChangesMade() noexcept {
 	return !table_manager.IsEmpty();
 }
@@ -209737,7 +209774,9 @@ public:
 	block_id_t GetFreeBlockId() override;
 	//! Returns whether or not a specified block is the root block
 	bool IsRootBlock(block_id_t root) override;
-	//! Mark a block as modified
+	//! Mark a block as free (immediately re-writeable)
+	void MarkBlockAsFree(block_id_t block_id) override;
+	//! Mark a block as modified (re-writeable after a checkpoint)
 	void MarkBlockAsModified(block_id_t block_id) override;
 	//! Increase the reference count of a block. The block should hold at least one reference
 	void IncreaseBlockReferenceCount(block_id_t block_id) override;
@@ -210035,6 +210074,14 @@ block_id_t SingleFileBlockManager::GetFreeBlockId() {
 	return block;
 }
 
+void SingleFileBlockManager::MarkBlockAsFree(block_id_t block_id) {
+	lock_guard<mutex> lock(block_lock);
+	D_ASSERT(block_id >= 0);
+	D_ASSERT(free_list.find(block_id) == free_list.end());
+	multi_use_blocks.erase(block_id);
+	free_list.insert(block_id);
+}
+
 void SingleFileBlockManager::MarkBlockAsModified(block_id_t block_id) {
 	lock_guard<mutex> lock(block_lock);
 	D_ASSERT(block_id >= 0);

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "42003b707e"
-#define DUCKDB_VERSION "v0.5.2-dev1460"
+#define DUCKDB_SOURCE_ID "4f495e8eea"
+#define DUCKDB_VERSION "v0.5.2-dev1468"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -25197,6 +25197,8 @@ public:
 	virtual block_id_t GetFreeBlockId() = 0;
 	//! Returns whether or not a specified block is the root block
 	virtual bool IsRootBlock(block_id_t root) = 0;
+	//! Mark a block as "free"; free blocks are immediately added to the free list and can be immediately overwritten
+	virtual void MarkBlockAsFree(block_id_t block_id) = 0;
 	//! Mark a block as "modified"; modified blocks are added to the free list after a checkpoint (i.e. their data is
 	//! assumed to be rewritten)
 	virtual void MarkBlockAsModified(block_id_t block_id) = 0;
@@ -26478,8 +26480,10 @@ public:
 	TableIndexList indexes;
 	//! The number of deleted rows
 	idx_t deleted_rows;
-	//! The optimistic data writer
+	//! The main optimistic data writer
 	OptimisticDataWriter optimistic_writer;
+	//! The set of all optimistic data writers associated with this table
+	vector<unique_ptr<OptimisticDataWriter>> optimistic_writers;
 
 public:
 	void InitializeScan(CollectionScanState &state, TableFilterSet *table_filters = nullptr);
@@ -26492,6 +26496,11 @@ public:
 
 	void AppendToIndexes(Transaction &transaction, TableAppendState &append_state, idx_t append_count,
 	                     bool append_to_table);
+	bool AppendToIndexes(Transaction &transaction, RowGroupCollection &source, TableIndexList &index_list,
+	                     const vector<LogicalType> &table_types, row_t &start_row);
+
+	//! Creates an optimistic writer for this table
+	OptimisticDataWriter *CreateOptimisticWriter();
 };
 
 class LocalTableManager {
@@ -26543,6 +26552,8 @@ public:
 	static void FinalizeAppend(LocalAppendState &state);
 	//! Merge a row group collection into the transaction-local storage
 	void LocalMerge(DataTable *table, RowGroupCollection &collection);
+	//! Create an optimistic writer for the specified table
+	OptimisticDataWriter *CreateOptimisticWriter(DataTable *table);
 
 	//! Delete a set of rows from the local storage
 	idx_t Delete(DataTable *table, Vector &row_ids, idx_t count);
@@ -26631,6 +26642,7 @@ class ClientContext;
 class ColumnDataCollection;
 class ColumnDefinition;
 class DataTable;
+class OptimisticDataWriter;
 class RowGroup;
 class StorageManager;
 class TableCatalogEntry;
@@ -26699,6 +26711,8 @@ public:
 	void LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection);
 	//! Merge a row group collection into the transaction-local storage
 	void LocalMerge(ClientContext &context, RowGroupCollection &collection);
+	//! Creates an optimistic writer for this table - used for optimistically writing parallel appends
+	OptimisticDataWriter *CreateOptimisticWriter(ClientContext &context);
 
 	//! Delete the entries with the specified row identifier from the table
 	idx_t Delete(TableCatalogEntry &table, ClientContext &context, Vector &row_ids, idx_t count);