duckdb 0.7.1-dev240.0 → 0.7.1-dev320.0

package/src/duckdb/src/storage/data_table.cpp

@@ -1144,15 +1144,78 @@ void DataTable::UpdateColumn(TableCatalogEntry &table, ClientContext &context, V
 }
 
 //===--------------------------------------------------------------------===//
-// Create Index Scan
+// Index Scan
 //===--------------------------------------------------------------------===//
-void DataTable::InitializeCreateIndexScan(CreateIndexScanState &state, const vector<column_t> &column_ids) {
+void DataTable::InitializeWALCreateIndexScan(CreateIndexScanState &state, const vector<column_t> &column_ids) {
 	// we grab the append lock to make sure nothing is appended until AFTER we finish the index scan
 	state.append_lock = std::unique_lock<mutex>(append_lock);
-	row_groups->InitializeCreateIndexScan(state);
 	InitializeScan(state, column_ids);
 }
 
+void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
+                            const vector<unique_ptr<Expression>> &expressions) {
+
+	// if the data table is empty
+	if (row_groups->IsEmpty()) {
+		info->indexes.AddIndex(std::move(index));
+		return;
+	}
+
+	auto &allocator = Allocator::Get(db);
+
+	DataChunk result;
+	result.Initialize(allocator, index->logical_types);
+
+	DataChunk intermediate;
+	vector<LogicalType> intermediate_types;
+	auto column_ids = index->column_ids;
+	column_ids.push_back(COLUMN_IDENTIFIER_ROW_ID);
+	for (auto &id : index->column_ids) {
+		auto &col = column_definitions[id];
+		intermediate_types.push_back(col.Type());
+	}
+	intermediate_types.emplace_back(LogicalType::ROW_TYPE);
+	intermediate.Initialize(allocator, intermediate_types);
+
+	// initialize an index scan
+	CreateIndexScanState state;
+	InitializeWALCreateIndexScan(state, column_ids);
+
+	if (!is_root) {
+		throw InternalException("Error during WAL replay. Cannot add an index to a table that has been altered.");
+	}
+
+	// now start incrementally building the index
+	{
+		IndexLock lock;
+		index->InitializeLock(lock);
+
+		while (true) {
+			intermediate.Reset();
+			result.Reset();
+			// scan a new chunk from the table to index
+			CreateIndexScan(state, intermediate, TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED);
+			if (intermediate.size() == 0) {
+				// finished scanning for index creation
+				// release all locks
+				break;
+			}
+			// resolve the expressions for this chunk
+			index->ExecuteExpressions(intermediate, result);
+
+			// insert into the index
+			if (!index->Insert(lock, result, intermediate.data[intermediate.ColumnCount() - 1])) {
+				throw InternalException("Error during WAL replay. Can't create unique index, table contains "
+				                        "duplicate data on indexed column(s).");
+			}
+		}
+	}
+	info->indexes.AddIndex(std::move(index));
+}
+
+//===--------------------------------------------------------------------===//
+// Statistics
+//===--------------------------------------------------------------------===//
 unique_ptr<BaseStatistics> DataTable::GetStatistics(ClientContext &context, column_t column_id) {
 	if (column_id == COLUMN_IDENTIFIER_ROW_ID) {
 		return nullptr;

package/src/duckdb/src/storage/index.cpp

@@ -86,7 +86,7 @@ bool Index::IndexIsUpdated(const vector<PhysicalIndex> &column_ids) const {
 	return false;
 }
 
-BlockPointer Index::Serialize(duckdb::MetaBlockWriter &writer) {
+BlockPointer Index::Serialize(MetaBlockWriter &writer) {
 	throw NotImplementedException("The implementation of this index serialization does not exist.");
 }
 

package/src/duckdb/src/storage/local_storage.cpp

@@ -127,7 +127,7 @@ LocalTableStorage::LocalTableStorage(DataTable &table)
 				unbound_expressions.push_back(expr->Copy());
 			}
 			indexes.AddIndex(make_unique<ART>(art.column_ids, art.table_io_manager, std::move(unbound_expressions),
-			                                  art.constraint_type, art.db, false));
+			                                  art.constraint_type, art.db, true));
 		}
 		return false;
 	});

package/src/duckdb/src/storage/table_index_list.cpp

@@ -60,8 +60,7 @@ void TableIndexList::VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, Data
 		throw InternalException("Internal Foreign Key error: could not find index to verify...");
 	}
 	conflict_manager.SetIndexCount(1);
-
-	index->LookupValues(chunk, conflict_manager);
+	index->CheckConstraintsForChunk(chunk, conflict_manager);
 }
 
 vector<column_t> TableIndexList::GetRequiredColumns() {

package/src/duckdb/src/storage/wal_replay.cpp

@@ -18,6 +18,8 @@
 #include "duckdb/storage/write_ahead_log.hpp"
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/main/attached_database.hpp"
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/catalog/catalog_entry/duck_index_entry.hpp"
 
 namespace duckdb {
 
@@ -154,6 +156,12 @@ void ReplayState::ReplayEntry(WALType entry_type) {
 	case WALType::DROP_TABLE_MACRO:
 		ReplayDropTableMacro();
 		break;
+	case WALType::CREATE_INDEX:
+		ReplayCreateIndex();
+		break;
+	case WALType::DROP_INDEX:
+		ReplayDropIndex();
+		break;
 	case WALType::USE_TABLE:
 		ReplayUseTable();
 		break;
@@ -379,6 +387,66 @@ void ReplayState::ReplayDropTableMacro() {
 	catalog.DropEntry(context, &info);
 }
 
+//===--------------------------------------------------------------------===//
+// Replay Index
+//===--------------------------------------------------------------------===//
+void ReplayState::ReplayCreateIndex() {
+
+	auto info = IndexCatalogEntry::Deserialize(source, context);
+	if (deserialize_only) {
+		return;
+	}
+
+	// get the physical table to which we'll add the index
+	auto table = catalog.GetEntry<TableCatalogEntry>(context, info->schema, info->table->table_name);
+	auto &data_table = table->GetStorage();
+
+	// bind the parsed expressions
+	if (info->expressions.empty()) {
+		for (auto &parsed_expr : info->parsed_expressions) {
+			info->expressions.push_back(parsed_expr->Copy());
+		}
+	}
+	auto binder = Binder::CreateBinder(context);
+	auto expressions = binder->BindCreateIndexExpressions(table, info.get());
+
+	// create the empty index
+	unique_ptr<Index> index;
+	switch (info->index_type) {
+	case IndexType::ART: {
+		index = make_unique<ART>(info->column_ids, TableIOManager::Get(data_table), expressions, info->constraint_type,
+		                         data_table.db, true);
+		break;
+	}
+	default:
+		throw InternalException("Unimplemented index type");
+	}
+
+	// add the index to the catalog
+	auto index_entry = (DuckIndexEntry *)catalog.CreateIndex(context, info.get());
+	index_entry->index = index.get();
+	index_entry->info = data_table.info;
+	for (auto &parsed_expr : info->parsed_expressions) {
+		index_entry->parsed_expressions.push_back(parsed_expr->Copy());
+	}
+
+	// physically add the index to the data table storage
+	data_table.WALAddIndex(context, std::move(index), expressions);
+}
+
+void ReplayState::ReplayDropIndex() {
+
+	DropInfo info;
+	info.type = CatalogType::INDEX_ENTRY;
+	info.schema = source.Read<string>();
+	info.name = source.Read<string>();
+	if (deserialize_only) {
+		return;
+	}
+
+	catalog.DropEntry(context, &info);
+}
+
 //===--------------------------------------------------------------------===//
 // Replay Data
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/storage/write_ahead_log.cpp

@@ -119,7 +119,7 @@ void WriteAheadLog::WriteSequenceValue(SequenceCatalogEntry *entry, SequenceValu
 }
 
 //===--------------------------------------------------------------------===//
-// MACRO'S
+// MACROS
 //===--------------------------------------------------------------------===//
 void WriteAheadLog::WriteCreateMacro(ScalarMacroCatalogEntry *entry) {
 	if (skip_writing) {
@@ -155,6 +155,26 @@ void WriteAheadLog::WriteDropTableMacro(TableMacroCatalogEntry *entry) {
 	writer->WriteString(entry->name);
 }
 
+//===--------------------------------------------------------------------===//
+// Indexes
+//===--------------------------------------------------------------------===//
+void WriteAheadLog::WriteCreateIndex(IndexCatalogEntry *entry) {
+	if (skip_writing) {
+		return;
+	}
+	writer->Write<WALType>(WALType::CREATE_INDEX);
+	entry->Serialize(*writer);
+}
+
+void WriteAheadLog::WriteDropIndex(IndexCatalogEntry *entry) {
+	if (skip_writing) {
+		return;
+	}
+	writer->Write<WALType>(WALType::DROP_INDEX);
+	writer->WriteString(entry->schema->name);
+	writer->WriteString(entry->name);
+}
+
 //===--------------------------------------------------------------------===//
 // Custom Types
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/transaction/commit_state.cpp

@@ -87,7 +87,9 @@ void CommitState::WriteCatalogEntry(CatalogEntry *entry, data_ptr_t dataptr) {
 	case CatalogType::TABLE_MACRO_ENTRY:
 		log->WriteCreateTableMacro((TableMacroCatalogEntry *)parent);
 		break;
-
+	case CatalogType::INDEX_ENTRY:
+		log->WriteCreateIndex((IndexCatalogEntry *)parent);
+		break;
 	case CatalogType::TYPE_ENTRY:
 		log->WriteCreateType((TypeCatalogEntry *)parent);
 		break;
@@ -119,6 +121,8 @@ void CommitState::WriteCatalogEntry(CatalogEntry *entry, data_ptr_t dataptr) {
 			log->WriteDropType((TypeCatalogEntry *)entry);
 			break;
 		case CatalogType::INDEX_ENTRY:
+			log->WriteDropIndex((IndexCatalogEntry *)entry);
+			break;
 		case CatalogType::PREPARED_STATEMENT:
 		case CatalogType::SCALAR_FUNCTION_ENTRY:
 			// do nothing, indexes/prepared statements/functions aren't persisted to disk
@@ -127,7 +131,6 @@ void CommitState::WriteCatalogEntry(CatalogEntry *entry, data_ptr_t dataptr) {
 			throw InternalException("Don't know how to drop this type!");
 		}
 		break;
-	case CatalogType::INDEX_ENTRY:
 	case CatalogType::PREPARED_STATEMENT:
 	case CatalogType::AGGREGATE_FUNCTION_ENTRY:
 	case CatalogType::SCALAR_FUNCTION_ENTRY:

package/src/duckdb/third_party/concurrentqueue/blockingconcurrentqueue.h

@@ -24,8 +24,8 @@ template<typename T, typename Traits = ConcurrentQueueDefaultTraits>
 class BlockingConcurrentQueue
 {
 private:
-	typedef ::duckdb_moodycamelmoodycamel::ConcurrentQueue<T, Traits> ConcurrentQueue;
-	typedef ::duckdb_moodycamelmoodycamel::LightweightSemaphore LightweightSemaphore;
+	typedef ::duckdb_moodycamel::ConcurrentQueue<T, Traits> ConcurrentQueue;
+	typedef ::duckdb_moodycamel::LightweightSemaphore LightweightSemaphore;
 
 public:
 	typedef typename ConcurrentQueue::producer_token_t producer_token_t;

package/src/statement.cpp

@@ -8,6 +8,8 @@
 #include <string>
 #include <regex>
 
+using std::unique_ptr;
+
 namespace node_duckdb {
 
 Napi::FunctionReference Statement::constructor;
@@ -28,13 +30,45 @@ Napi::Object Statement::Init(Napi::Env env, Napi::Object exports) {
 	return exports;
 }
 
+static unique_ptr<duckdb::PreparedStatement> PrepareManyInternal(Statement &statement) {
+	auto &connection = statement.connection_ref->connection;
+	std::vector<unique_ptr<duckdb::SQLStatement>> statements;
+	try {
+		if (connection == nullptr) {
+			throw duckdb::ConnectionException("Connection was never established or has been closed already");
+		}
+
+		// Prepare all statements
+		statements = connection->ExtractStatements(statement.sql);
+		if (statements.empty()) {
+			throw duckdb::InvalidInputException("No statement to prepare!");
+		}
+
+		// if there are multiple statements, we directly execute the statements besides the last one
+		// we only return the result of the last statement to the user, unless one of the previous statements fails
+		for (idx_t i = 0; i + 1 < statements.size(); i++) {
+			auto pending_query = connection->PendingQuery(std::move(statements[i]));
+			auto res = pending_query->Execute();
+			if (res->HasError()) {
+				return duckdb::make_unique<duckdb::PreparedStatement>(res->GetErrorObject());
+			}
+		}
+
+		return connection->Prepare(std::move(statements.back()));
+	} catch (const duckdb::Exception &ex) {
+		return duckdb::make_unique<duckdb::PreparedStatement>(duckdb::PreservedError(ex));
+	} catch (std::exception &ex) {
+		return duckdb::make_unique<duckdb::PreparedStatement>(duckdb::PreservedError(ex));
+	}
+}
+
 struct PrepareTask : public Task {
 	PrepareTask(Statement &statement, Napi::Function callback) : Task(statement, callback) {
 	}
 
 	void DoWork() override {
 		auto &statement = Get<Statement>();
-		statement.statement = statement.connection_ref->connection->Prepare(statement.sql);
+		statement.statement = PrepareManyInternal(statement);
 	}
 
 	void Callback() override {
@@ -221,7 +255,7 @@ struct StatementParam {
 };
 
 struct RunPreparedTask : public Task {
-	RunPreparedTask(Statement &statement, duckdb::unique_ptr<StatementParam> params, RunType run_type)
+	RunPreparedTask(Statement &statement, unique_ptr<StatementParam> params, RunType run_type)
 	    : Task(statement, params->callback), params(std::move(params)), run_type(run_type) {
 	}
 
@@ -366,13 +400,13 @@ struct RunPreparedTask : public Task {
 		} break;
 		}
 	}
-	std::unique_ptr<duckdb::QueryResult> result;
-	duckdb::unique_ptr<StatementParam> params;
+	unique_ptr<duckdb::QueryResult> result;
+	unique_ptr<StatementParam> params;
 	RunType run_type;
 };
 
 struct RunQueryTask : public Task {
-	RunQueryTask(Statement &statement, duckdb::unique_ptr<StatementParam> params, Napi::Promise::Deferred deferred)
+	RunQueryTask(Statement &statement, unique_ptr<StatementParam> params, Napi::Promise::Deferred deferred)
 	    : Task(statement), deferred(deferred), params(std::move(params)) {
 	}
 
@@ -406,11 +440,11 @@ struct RunQueryTask : public Task {
 	}
 
 	Napi::Promise::Deferred deferred;
-	std::unique_ptr<duckdb::QueryResult> result;
-	duckdb::unique_ptr<StatementParam> params;
+	unique_ptr<duckdb::QueryResult> result;
+	unique_ptr<StatementParam> params;
 };
 
-duckdb::unique_ptr<StatementParam> Statement::HandleArgs(const Napi::CallbackInfo &info) {
+unique_ptr<StatementParam> Statement::HandleArgs(const Napi::CallbackInfo &info) {
 	size_t start_idx = ignore_first_param ? 1 : 0;
 	auto params = duckdb::make_unique<StatementParam>();
 
@@ -541,7 +575,7 @@ struct GetChunkTask : public Task {
 	}
 
 	Napi::Promise::Deferred deferred;
-	std::unique_ptr<duckdb::DataChunk> chunk;
+	unique_ptr<duckdb::DataChunk> chunk;
 };
 
 struct GetNextArrowIpcTask : public Task {
@@ -571,10 +605,10 @@ struct GetNextArrowIpcTask : public Task {
 		duckdb::string_t blob = *(duckdb::string_t *)(chunk->data[0].GetData());
 
 		// Transfer ownership and Construct ArrayBuffer
-		auto data_chunk_ptr = new std::unique_ptr<duckdb::DataChunk>();
+		auto data_chunk_ptr = new unique_ptr<duckdb::DataChunk>();
 		*data_chunk_ptr = std::move(chunk);
 		auto deleter = [](Napi::Env, void *finalizeData, void *hint) {
-			delete static_cast<std::unique_ptr<duckdb::DataChunk> *>(hint);
+			delete static_cast<unique_ptr<duckdb::DataChunk> *>(hint);
 		};
 		auto array_buffer =
 		    Napi::ArrayBuffer::New(env, (void *)blob.GetDataUnsafe(), blob.GetSize(), deleter, data_chunk_ptr);
@@ -583,7 +617,7 @@ struct GetNextArrowIpcTask : public Task {
 	}
 
 	Napi::Promise::Deferred deferred;
-	std::unique_ptr<duckdb::DataChunk> chunk;
+	unique_ptr<duckdb::DataChunk> chunk;
 };
 
 Napi::Value QueryResult::NextChunk(const Napi::CallbackInfo &info) {

package/test/arrow.test.ts

@@ -20,13 +20,13 @@ describe('arrow IPC API fails neatly when extension not loaded', function() {
         db.arrowIPCStream(query).then(
             () => Promise.reject(new Error('Expected method to reject.')),
             err => {
-                assert.ok(err.message.includes("Catalog Error: Function with name to_arrow_ipc is not on the catalog, but it exists in the arrow extension. To Install and Load the extension, run: INSTALL arrow; LOAD arrow;"))
+                assert.ok(err.message.includes("arrow"))
             }
         );
 
         db.arrowIPCAll(`SELECT * FROM ipc_table`, function (err: null | Error, result: ArrowArray) {
             if (err) {
-                assert.ok(err.message.includes("Catalog Error: Function with name to_arrow_ipc is not on the catalog, but it exists in the arrow extension. To Install and Load the extension, run: INSTALL arrow; LOAD arrow;"))
+                assert.ok(err.message.includes("arrow"))
             } else {
                 assert.fail("Expected error");
             }
@@ -39,7 +39,7 @@ describe('arrow IPC API fails neatly when extension not loaded', function() {
     it('register buffer should be disabled currently', function(done) {
         db.register_buffer("test", [new Uint8Array(new ArrayBuffer(10))], true, (err: null | Error) => {
             assert.ok(err)
-            assert.ok(err.toString().includes("Function with name scan_arrow_ipc is not on the catalog, but it exists in the arrow extension. To Install and Load the extension, run: INSTALL arrow; LOAD arrow;"));
+            assert.ok(err.toString().includes("arrow"));
             done()
         });
     });

package/test/prepare.test.ts

@@ -66,7 +66,6 @@ describe('prepare', function() {
             }
         });
 
-
         it('should prepare a statement and return values again', function(done) {
             var stmt = db.prepare("SELECT txt, num, flt, blb FROM foo ORDER BY num", function(err: null | Error) {
                 if (err) throw err;
@@ -343,6 +342,45 @@ describe('prepare', function() {
         after(function(done) { db.close(done); });
     });
 
+    describe('prepare multiple statements', function() {
+        var db: sqlite3.Database;
+        before(function(done) { db = new sqlite3.Database(':memory:',
+            function(err) {
+                db.run("CREATE TABLE foo (a integer)", done)
+            }
+            ); });
+
+        it('should directly execute first statements', function(done) {
+            db.prepare("insert into foo values (3); insert into foo values (4); select * from foo")
+                .all(function(err: null | Error, rows: TableData) {
+                    if (err) throw err;
+                    assert.equal(rows[0].a, 3);
+                    assert.equal(rows[1].a, 4);
+                })
+                .finalize(done);
+        });
+
+        it('should fail in prepare, when executing the first statement', function(done) {
+            let prepared = db.prepare("SELECT * FROM non_existent_table; SELECT 42", function(err: null | Error) {
+				if (err) {
+					done();
+					return;
+				}
+			});
+        });
+
+        it('should fail in prepare, when preparing the first statement', function(done) {
+            let prepared = db.prepare("SELCET * FROM foo; SELECT 42", function(err: null | Error) {
+				if (err) {
+					done();
+					return;
+				}
+			});
+        });
+
+        after(function(done) { db.close(done); });
+    });
+
     describe('all()', function() {
         var db: sqlite3.Database;
         before(function(done) { db = new sqlite3.Database(':memory:',

package/test/typescript_decls.test.ts

@@ -2,7 +2,7 @@ import * as duckdb from "..";
 import assert from "assert";
 import fs from "fs";
 
-describe("TypeScript declarataions", function () {
+describe("TypeScript declarations", function () {
   var db: duckdb.Database;
   before(function (done) {
     db = new duckdb.Database(":memory:", duckdb.OPEN_READWRITE, done);