duckdb 0.6.1-dev86.0 → 0.6.2-dev13.0

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "0c8bafb821"
-#define DUCKDB_VERSION "v0.6.1-dev86"
+#define DUCKDB_SOURCE_ID "afeda73278"
+#define DUCKDB_VERSION "v0.6.2-dev13"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -4351,6 +4351,7 @@ void DestroyObject(T *ptr) {
 //! As such this class should be used primarily for larger allocations.
 struct BufferAllocator {
 	DUCKDB_API static Allocator &Get(ClientContext &context);
+	DUCKDB_API static Allocator &Get(DatabaseInstance &db);
 };
 
 } // namespace duckdb
@@ -4404,7 +4405,7 @@ namespace duckdb {
 //! returned pointer will remain valid until the StringHeap is destroyed
 class StringHeap {
 public:
-	StringHeap();
+	StringHeap(Allocator &allocator = Allocator::DefaultAllocator());
 
 	void Destroy();
 	void Move(StringHeap &other);
@@ -9171,6 +9172,7 @@ public:
 
 public:
 	virtual unique_ptr<CatalogEntry> AlterEntry(ClientContext &context, AlterInfo *info);
+	virtual void UndoAlter(ClientContext &context, AlterInfo *info);
 
 	virtual unique_ptr<CatalogEntry> Copy(ClientContext &context);
 
@@ -11162,6 +11164,8 @@ public:
 public:
 	bool HasGeneratedColumns() const;
 	unique_ptr<CatalogEntry> AlterEntry(ClientContext &context, AlterInfo *info) override;
+	void UndoAlter(ClientContext &context, AlterInfo *info) override;
+
 	//! Returns whether or not a column with the given name exists
 	DUCKDB_API bool ColumnExists(const string &name);
 	//! Returns a reference to the column of the specified name. Throws an
@@ -11340,6 +11344,9 @@ public:
 		return true;
 	}
 
+	//! Returns the set of table indexes of this operator
+	virtual vector<idx_t> GetTableIndex() const;
+
 protected:
 	//! Resolve types for this specific operator
 	virtual void ResolveTypes() = 0;
@@ -14190,6 +14197,9 @@ public:
 	void Reset();
 	void ResetSink();
 	void ResetSource(bool force);
+	void ClearSource() {
+		source_state.reset();
+	}
 	void Schedule(shared_ptr<Event> &event);
 
 	//! Finalize this pipeline
@@ -15527,27 +15537,46 @@ namespace duckdb {
 struct AlterInfo;
 
 class ClientContext;
+struct MappingValue;
+struct EntryIndex;
 
 typedef unordered_map<CatalogSet *, unique_lock<mutex>> set_lock_map_t;
 
-struct MappingValue {
-	explicit MappingValue(idx_t index_) : index(index_), timestamp(0), deleted(false), parent(nullptr) {
+struct EntryValue {
+	EntryValue() {
+		throw InternalException("EntryValue called without a catalog entry");
 	}
 
-	idx_t index;
-	transaction_t timestamp;
-	bool deleted;
-	unique_ptr<MappingValue> child;
-	MappingValue *parent;
+	explicit EntryValue(unique_ptr<CatalogEntry> entry_p) : entry(move(entry_p)), reference_count(0) {
+	}
+	//! enable move constructors
+	EntryValue(EntryValue &&other) noexcept {
+		Swap(other);
+	}
+	EntryValue &operator=(EntryValue &&other) noexcept {
+		Swap(other);
+		return *this;
+	}
+	void Swap(EntryValue &other) {
+		std::swap(entry, other.entry);
+		idx_t count = reference_count;
+		reference_count = other.reference_count.load();
+		other.reference_count = count;
+	}
+
+	unique_ptr<CatalogEntry> entry;
+	atomic<idx_t> reference_count;
 };
 
 //! The Catalog Set stores (key, value) map of a set of CatalogEntries
 class CatalogSet {
 	friend class DependencyManager;
 	friend class EntryDropper;
+	friend struct EntryIndex;
 
 public:
 	DUCKDB_API explicit CatalogSet(Catalog &catalog, unique_ptr<DefaultGenerator> defaults = nullptr);
+	~CatalogSet();
 
 	//! Create an entry in the catalog set. Returns whether or not it was
 	//! successful.
@@ -15588,7 +15617,6 @@ public:
 	DUCKDB_API static bool HasConflict(ClientContext &context, transaction_t timestamp);
 	DUCKDB_API static bool UseTimestamp(ClientContext &context, transaction_t timestamp);
 
-	CatalogEntry *GetEntryFromIndex(idx_t index);
 	void UpdateTimestamp(CatalogEntry *entry, transaction_t timestamp);
 
 private:
@@ -15601,29 +15629,32 @@ private:
 	//! Given a root entry, gets the entry valid for this transaction
 	CatalogEntry *GetEntryForTransaction(ClientContext &context, CatalogEntry *current);
 	CatalogEntry *GetCommittedEntry(CatalogEntry *current);
-	bool GetEntryInternal(ClientContext &context, const string &name, idx_t &entry_index, CatalogEntry *&entry);
-	bool GetEntryInternal(ClientContext &context, idx_t entry_index, CatalogEntry *&entry);
+	bool GetEntryInternal(ClientContext &context, const string &name, EntryIndex *entry_index, CatalogEntry *&entry);
+	bool GetEntryInternal(ClientContext &context, EntryIndex &entry_index, CatalogEntry *&entry);
 	//! Drops an entry from the catalog set; must hold the catalog_lock to safely call this
-	void DropEntryInternal(ClientContext &context, idx_t entry_index, CatalogEntry &entry, bool cascade);
+	void DropEntryInternal(ClientContext &context, EntryIndex entry_index, CatalogEntry &entry, bool cascade);
 	CatalogEntry *CreateEntryInternal(ClientContext &context, unique_ptr<CatalogEntry> entry);
 	MappingValue *GetMapping(ClientContext &context, const string &name, bool get_latest = false);
-	void PutMapping(ClientContext &context, const string &name, idx_t entry_index);
+	void PutMapping(ClientContext &context, const string &name, EntryIndex entry_index);
 	void DeleteMapping(ClientContext &context, const string &name);
-	void DropEntryDependencies(ClientContext &context, idx_t entry_index, CatalogEntry &entry, bool cascade);
+	void DropEntryDependencies(ClientContext &context, EntryIndex &entry_index, CatalogEntry &entry, bool cascade);
 
 	//! Create all default entries
 	void CreateDefaultEntries(ClientContext &context, unique_lock<mutex> &lock);
 	//! Attempt to create a default entry with the specified name. Returns the entry if successful, nullptr otherwise.
 	CatalogEntry *CreateDefaultEntry(ClientContext &context, const string &name, unique_lock<mutex> &lock);
 
+	EntryIndex PutEntry(idx_t entry_index, unique_ptr<CatalogEntry> entry);
+	void PutEntry(EntryIndex index, unique_ptr<CatalogEntry> entry);
+
 private:
 	Catalog &catalog;
 	//! The catalog lock is used to make changes to the data
 	mutex catalog_lock;
+	//! The set of catalog entries
+	unordered_map<idx_t, EntryValue> entries;
 	//! Mapping of string to catalog entry
 	case_insensitive_map_t<unique_ptr<MappingValue>> mapping;
-	//! The set of catalog entries
-	unordered_map<idx_t, unique_ptr<CatalogEntry>> entries;
 	//! The current catalog entry index
 	idx_t current_entry = 0;
 	//! The generator used to generate default internal entries
@@ -16616,6 +16647,14 @@ public:
 	DUCKDB_API shared_ptr<Relation> RelationFromQuery(unique_ptr<SelectStatement> select_stmt,
 	                                                  const string &alias = "queryrelation");
 
+	//! Returns a substrait BLOB from a valid query
+	DUCKDB_API string GetSubstrait(const string &query);
+	//! Returns a Query Result from a substrait blob
+	DUCKDB_API unique_ptr<QueryResult> FromSubstrait(const string &proto);
+	//! Returns a substrait BLOB from a valid query
+	DUCKDB_API string GetSubstraitJSON(const string &query);
+	//! Returns a Query Result from a substrait JSON
+	DUCKDB_API unique_ptr<QueryResult> FromSubstraitJSON(const string &json);
 	DUCKDB_API void BeginTransaction();
 	DUCKDB_API void Commit();
 	DUCKDB_API void Rollback();
@@ -18911,6 +18950,9 @@ struct OperatorExtensionInfo {
 typedef BoundStatement (*bind_function_t)(ClientContext &context, Binder &binder, OperatorExtensionInfo *info,
                                           SQLStatement &statement);
 
+// forward declaration to avoid circular reference
+struct LogicalExtensionOperator;
+
 class OperatorExtension {
 public:
 	bind_function_t Bind;
@@ -18918,6 +18960,10 @@ public:
 	//! Additional info passed to the CreatePlan & Bind functions
 	shared_ptr<OperatorExtensionInfo> operator_info;
 
+	virtual std::string GetName() = 0;
+	virtual std::unique_ptr<LogicalExtensionOperator> Deserialize(LogicalDeserializationState &state,
+	                                                              FieldReader &reader) = 0;
+
 	DUCKDB_API virtual ~OperatorExtension() {
 	}
 };
@@ -19063,7 +19109,7 @@ public:
 	//! A reference to the (shared) default allocator (Allocator::DefaultAllocator)
 	shared_ptr<Allocator> default_allocator;
 	//! Extensions made to binder
-	vector<OperatorExtension> operator_extensions;
+	vector<std::unique_ptr<OperatorExtension>> operator_extensions;
 
 public:
 	DUCKDB_API static DBConfig &GetConfig(ClientContext &context);
@@ -25993,6 +26039,7 @@ struct SBIterator {
 
 
 
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -26164,6 +26211,10 @@ public:
 		can_destroy = can_destroy_p;
 	}
 
+	inline const idx_t &GetMemoryUsage() const {
+		return memory_usage;
+	}
+
 private:
 	static BufferHandle Load(shared_ptr<BlockHandle> &handle, unique_ptr<FileBuffer> buffer = nullptr);
 	unique_ptr<FileBuffer> UnloadAndTakeBlock();
@@ -26197,7 +26248,6 @@ private:
 
 
 
-
 namespace duckdb {
 class BlockManager;
 class DatabaseInstance;
@@ -26218,10 +26268,6 @@ public:
 	BufferManager(DatabaseInstance &db, string temp_directory, idx_t maximum_memory);
 	virtual ~BufferManager();
 
-	//! Register an in-memory buffer of arbitrary size, as long as it is >= BLOCK_SIZE. can_destroy signifies whether or
-	//! not the buffer can be destroyed when unpinned, or whether or not it needs to be written to a temporary file so
-	//! it can be reloaded. The resulting buffer will already be allocated, but needs to be pinned in order to be used.
-	shared_ptr<BlockHandle> RegisterMemory(idx_t block_size, bool can_destroy);
 	//! Registers an in-memory buffer that cannot be unloaded until it is destroyed
 	//! This buffer can be small (smaller than BLOCK_SIZE)
 	//! Unpin and pin are nops on this block of memory
@@ -26229,7 +26275,8 @@ public:
 
 	//! Allocate an in-memory buffer with a single pin.
 	//! The allocated memory is released when the buffer handle is destroyed.
-	DUCKDB_API BufferHandle Allocate(idx_t block_size);
+	DUCKDB_API BufferHandle Allocate(idx_t block_size, bool can_destroy = true,
+	                                 shared_ptr<BlockHandle> *block = nullptr);
 
 	//! Reallocate an in-memory buffer that is pinned.
 	void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size);
@@ -26263,6 +26310,10 @@ public:
 		return db;
 	}
 
+	static idx_t GetAllocSize(idx_t block_size) {
+		return AlignValue<idx_t, Storage::SECTOR_SIZE>(block_size + Storage::BLOCK_HEADER_SIZE);
+	}
+
 	//! Construct a managed buffer.
 	//! The block_id is just used for internal tracking. It doesn't map to any actual
 	//! BlockManager.
@@ -26273,6 +26324,12 @@ public:
 	DUCKDB_API void FreeReservedMemory(idx_t size);
 
 private:
+	//! Register an in-memory buffer of arbitrary size, as long as it is >= BLOCK_SIZE. can_destroy signifies whether or
+	//! not the buffer can be destroyed when unpinned, or whether or not it needs to be written to a temporary file so
+	//! it can be reloaded. The resulting buffer will already be allocated, but needs to be pinned in order to be used.
+	//! This needs to be private to prevent creating blocks without ever pinning them:
+	//! blocks that are never pinned are never added to the eviction queue
+	shared_ptr<BlockHandle> RegisterMemory(idx_t block_size, bool can_destroy);
 	//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
 	//! (i.e. not enough blocks could be evicted)
 	//! If the "buffer" argument is specified AND the system can find a buffer to re-use for the given allocation size
@@ -26354,7 +26411,8 @@ public:
 	RowDataBlock(BufferManager &buffer_manager, idx_t capacity, idx_t entry_size)
 	    : capacity(capacity), entry_size(entry_size), count(0), byte_offset(0) {
 		idx_t size = MaxValue<idx_t>(Storage::BLOCK_SIZE, capacity * entry_size);
-		block = buffer_manager.RegisterMemory(size, false);
+		buffer_manager.Allocate(size, false, &block);
+		D_ASSERT(BufferManager::GetAllocSize(size) == block->GetMemoryUsage());
 	}
 	explicit RowDataBlock(idx_t entry_size) : entry_size(entry_size) {
 	}
@@ -26428,17 +26486,23 @@ public:
 		count = 0;
 	}
 
-	//! The size (in bytes) of this RowDataCollection if it were stored in a single block
+	//! The size (in bytes) of this RowDataCollection
 	idx_t SizeInBytes() const {
-		idx_t bytes = 0;
-		if (entry_size == 1) {
-			for (auto &block : blocks) {
-				bytes += block->byte_offset;
-			}
-		} else {
-			bytes = count * entry_size;
+		VerifyBlockSizes();
+		idx_t size = 0;
+		for (auto &block : blocks) {
+			size += block->block->GetMemoryUsage();
 		}
-		return bytes;
+		return size;
+	}
+
+	//! Verifies that the block sizes are correct (Debug only)
+	void VerifyBlockSizes() const {
+#ifdef DEBUG
+		for (auto &block : blocks) {
+			D_ASSERT(block->block->GetMemoryUsage() == BufferManager::GetAllocSize(block->capacity * entry_size));
+		}
+#endif
 	}
 
 	static inline idx_t EntriesPerBlock(idx_t width) {
@@ -30777,6 +30841,7 @@ public:
 
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	vector<idx_t> GetTableIndex() const override;
 
 protected:
 	void ResolveTypes() override;

package/src/duckdb_node.hpp

@@ -133,7 +133,6 @@ public:
 	std::unique_ptr<duckdb::Connection> connection;
 	Database *database_ref;
 	std::unordered_map<std::string, duckdb_node_udf_function_t> udfs;
-	std::unordered_map<std::string, std::vector<std::pair<uint64_t, uint64_t>>> buffers;
 	std::unordered_map<std::string, Napi::Reference<Napi::Array>> array_references;
 };