duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/include/duckdb/function/window/window_value_function.hpp

@@ -15,14 +15,14 @@ namespace duckdb {
 // Base class for non-aggregate functions that have a payload
 class WindowValueExecutor : public WindowExecutor {
 public:
-	WindowValueExecutor(BoundWindowExpression &wexpr, ClientContext &context, WindowSharedExpressions &shared);
+	WindowValueExecutor(BoundWindowExpression &wexpr, WindowSharedExpressions &shared);
 
-	void Finalize(WindowExecutorGlobalState &gstate, WindowExecutorLocalState &lstate,
-	              CollectionPtr collection) const override;
+	void Finalize(ExecutionContext &context, CollectionPtr collection, OperatorSinkInput &sink) const override;
 
-	unique_ptr<WindowExecutorGlobalState> GetGlobalState(const idx_t payload_count, const ValidityMask &partition_mask,
-	                                                     const ValidityMask &order_mask) const override;
-	unique_ptr<WindowExecutorLocalState> GetLocalState(const WindowExecutorGlobalState &gstate) const override;
+	unique_ptr<GlobalSinkState> GetGlobalState(ClientContext &client, const idx_t payload_count,
+	                                           const ValidityMask &partition_mask,
+	                                           const ValidityMask &order_mask) const override;
+	unique_ptr<LocalSinkState> GetLocalState(ExecutionContext &context, const GlobalSinkState &gstate) const override;
 
 	//! The column index of the value column
 	column_t child_idx = DConstants::INVALID_INDEX;
@@ -38,42 +38,65 @@ public:
 
 class WindowLeadLagExecutor : public WindowValueExecutor {
 public:
-	WindowLeadLagExecutor(BoundWindowExpression &wexpr, ClientContext &context, WindowSharedExpressions &shared);
+	WindowLeadLagExecutor(BoundWindowExpression &wexpr, WindowSharedExpressions &shared);
 
-	unique_ptr<WindowExecutorGlobalState> GetGlobalState(const idx_t payload_count, const ValidityMask &partition_mask,
-	                                                     const ValidityMask &order_mask) const override;
-	unique_ptr<WindowExecutorLocalState> GetLocalState(const WindowExecutorGlobalState &gstate) const override;
+	unique_ptr<GlobalSinkState> GetGlobalState(ClientContext &client, const idx_t payload_count,
+	                                           const ValidityMask &partition_mask,
+	                                           const ValidityMask &order_mask) const override;
+	unique_ptr<LocalSinkState> GetLocalState(ExecutionContext &context, const GlobalSinkState &gstate) const override;
 
 protected:
-	void EvaluateInternal(WindowExecutorGlobalState &gstate, WindowExecutorLocalState &lstate, DataChunk &eval_chunk,
-	                      Vector &result, idx_t count, idx_t row_idx) const override;
+	void EvaluateInternal(ExecutionContext &context, DataChunk &eval_chunk, Vector &result, idx_t count, idx_t row_idx,
+	                      OperatorSinkInput &sink) const override;
 };
 
 class WindowFirstValueExecutor : public WindowValueExecutor {
 public:
-	WindowFirstValueExecutor(BoundWindowExpression &wexpr, ClientContext &context, WindowSharedExpressions &shared);
+	WindowFirstValueExecutor(BoundWindowExpression &wexpr, WindowSharedExpressions &shared);
 
 protected:
-	void EvaluateInternal(WindowExecutorGlobalState &gstate, WindowExecutorLocalState &lstate, DataChunk &eval_chunk,
-	                      Vector &result, idx_t count, idx_t row_idx) const override;
+	void EvaluateInternal(ExecutionContext &context, DataChunk &eval_chunk, Vector &result, idx_t count, idx_t row_idx,
+	                      OperatorSinkInput &sink) const override;
 };
 
 class WindowLastValueExecutor : public WindowValueExecutor {
 public:
-	WindowLastValueExecutor(BoundWindowExpression &wexpr, ClientContext &context, WindowSharedExpressions &shared);
+	WindowLastValueExecutor(BoundWindowExpression &wexpr, WindowSharedExpressions &shared);
 
 protected:
-	void EvaluateInternal(WindowExecutorGlobalState &gstate, WindowExecutorLocalState &lstate, DataChunk &eval_chunk,
-	                      Vector &result, idx_t count, idx_t row_idx) const override;
+	void EvaluateInternal(ExecutionContext &context, DataChunk &eval_chunk, Vector &result, idx_t count, idx_t row_idx,
+	                      OperatorSinkInput &sink) const override;
 };
 
 class WindowNthValueExecutor : public WindowValueExecutor {
 public:
-	WindowNthValueExecutor(BoundWindowExpression &wexpr, ClientContext &context, WindowSharedExpressions &shared);
+	WindowNthValueExecutor(BoundWindowExpression &wexpr, WindowSharedExpressions &shared);
 
 protected:
-	void EvaluateInternal(WindowExecutorGlobalState &gstate, WindowExecutorLocalState &lstate, DataChunk &eval_chunk,
-	                      Vector &result, idx_t count, idx_t row_idx) const override;
+	void EvaluateInternal(ExecutionContext &context, DataChunk &eval_chunk, Vector &result, idx_t count, idx_t row_idx,
+	                      OperatorSinkInput &sink) const override;
+};
+
+class WindowFillExecutor : public WindowValueExecutor {
+public:
+	WindowFillExecutor(BoundWindowExpression &wexpr, WindowSharedExpressions &shared);
+
+	//! Never ignore nulls (that's the point!)
+	bool IgnoreNulls() const override {
+		return false;
+	}
+
+	unique_ptr<GlobalSinkState> GetGlobalState(ClientContext &client, const idx_t payload_count,
+	                                           const ValidityMask &partition_mask,
+	                                           const ValidityMask &order_mask) const override;
+	unique_ptr<LocalSinkState> GetLocalState(ExecutionContext &context, const GlobalSinkState &gstate) const override;
+
+	//! Secondary order collection index
+	idx_t order_idx = DConstants::INVALID_INDEX;
+
+protected:
+	void EvaluateInternal(ExecutionContext &context, DataChunk &eval_chunk, Vector &result, idx_t count, idx_t row_idx,
+	                      OperatorSinkInput &sink) const override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/logging/log_manager.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/logging/log_storage.hpp
+// duckdb/logging/log_manager.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/logging/logger.hpp"
+#include "duckdb/logging/log_storage.hpp"
 #include "duckdb/common/types/timestamp.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 
@@ -47,15 +48,18 @@ public:
 
 	//! Get a shared_ptr to the log storage (For example, to scan it)
 	DUCKDB_API shared_ptr<LogStorage> GetLogStorage();
-	DUCKDB_API bool CanScan();
+	DUCKDB_API bool CanScan(LoggingTargetTable table);
 
+	DUCKDB_API void SetConfig(DatabaseInstance &db, const LogConfig &config);
 	DUCKDB_API void SetEnableLogging(bool enable);
 	DUCKDB_API void SetLogMode(LogMode mode);
 	DUCKDB_API void SetLogLevel(LogLevel level);
-	DUCKDB_API void SetEnabledLogTypes(unordered_set<string> &enabled_log_types);
-	DUCKDB_API void SetDisabledLogTypes(unordered_set<string> &disabled_log_types);
+	DUCKDB_API void SetEnabledLogTypes(optional_ptr<unordered_set<string>> enabled_log_types);
+	DUCKDB_API void SetDisabledLogTypes(optional_ptr<unordered_set<string>> disabled_log_types);
 	DUCKDB_API void SetLogStorage(DatabaseInstance &db, const string &storage_name);
 
+	DUCKDB_API void UpdateLogStorageConfig(DatabaseInstance &db, case_insensitive_map_t<Value> &config_value);
+
 	DUCKDB_API void SetEnableStructuredLoggers(vector<string> &enabled_logger_types);
 
 	DUCKDB_API void TruncateLogStorage();
@@ -75,8 +79,12 @@ protected:
 	// This allows efficiently pushing a cached set of log entries into the log manager
 	void FlushCachedLogEntries(DataChunk &chunk, const RegisteredLoggingContext &context);
 
+	void SetLogStorageInternal(DatabaseInstance &db, const string &storage_name);
+
 	optional_ptr<const LogType> LookupLogTypeInternal(const string &type);
 
+	void SetConfigInternal(LogConfig config);
+
 	mutex lock;
 	LogConfig config;
 

package/src/duckdb/src/include/duckdb/logging/log_storage.hpp

@@ -13,6 +13,8 @@
 #include "duckdb/common/optional_idx.hpp"
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/unordered_set.hpp"
+#include "duckdb/common/serializer/write_stream.hpp"
+#include "duckdb/common/serializer/buffered_file_writer.hpp"
 
 #include "duckdb/common/types/column/column_data_scan_states.hpp"
 #include "duckdb/parallel/thread_context.hpp"
@@ -21,9 +23,26 @@ namespace duckdb {
 struct RegisteredLoggingContext;
 class ColumnDataCollection;
 struct ColumnDataScanState;
+class MemoryStream;
+struct LogStorageConfig;
+class CSVWriter;
+struct CSVWriterState;
+class BufferedFileWriter;
+struct CSVWriterOptions;
+struct CSVReaderOptions;
+
+//! Logging storage can store entries normalized or denormalized. This enum describes what a single table/file/etc
+//! contains
+enum class LoggingTargetTable {
+	ALL_LOGS,     // Denormalized: log entries consisting of both the full log entry and the context
+	LOG_ENTRIES,  // Normalized: contains only the log entries and a context_id
+	LOG_CONTEXTS, // Normalized: contains only the log contexts
+};
 
 class LogStorageScanState {
 public:
+	explicit LogStorageScanState(LoggingTargetTable table_p) : table(table_p) {
+	}
 	virtual ~LogStorageScanState() = default;
 
 	template <class TARGET>
@@ -36,99 +55,301 @@ public:
 		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<const TARGET &>(*this);
 	}
+
+	LoggingTargetTable table;
 };
 
-// Interface for writing log entries
+// Interface for Log Storage
 class LogStorage {
 public:
 	DUCKDB_API explicit LogStorage() {
 	}
 	DUCKDB_API virtual ~LogStorage() = default;
 
+	virtual const string GetStorageName() = 0;
+
+	static vector<LogicalType> GetSchema(LoggingTargetTable table);
+	static vector<string> GetColumnNames(LoggingTargetTable table);
+
 	//! WRITING
 	DUCKDB_API virtual void WriteLogEntry(timestamp_t timestamp, LogLevel level, const string &log_type,
 	                                      const string &log_message, const RegisteredLoggingContext &context) = 0;
 	DUCKDB_API virtual void WriteLogEntries(DataChunk &chunk, const RegisteredLoggingContext &context) = 0;
-	DUCKDB_API virtual void Flush() = 0;
+	DUCKDB_API virtual void FlushAll() = 0;
+	DUCKDB_API virtual void Flush(LoggingTargetTable table) = 0;
+	DUCKDB_API virtual void Truncate();
+	DUCKDB_API virtual bool IsEnabled(LoggingTargetTable table) = 0;
 
 	//! READING (OPTIONAL)
-	DUCKDB_API virtual bool CanScan() {
+	DUCKDB_API virtual bool CanScan(LoggingTargetTable table) {
 		return false;
 	}
-	DUCKDB_API virtual unique_ptr<LogStorageScanState> CreateScanEntriesState() const;
-	DUCKDB_API virtual bool ScanEntries(LogStorageScanState &state, DataChunk &result) const;
-	DUCKDB_API virtual void InitializeScanEntries(LogStorageScanState &state) const;
-	DUCKDB_API virtual unique_ptr<LogStorageScanState> CreateScanContextsState() const;
-	DUCKDB_API virtual bool ScanContexts(LogStorageScanState &state, DataChunk &result) const;
-	DUCKDB_API virtual void InitializeScanContexts(LogStorageScanState &state) const;
+	// Reading interface 1: basic single-threaded scan
+	DUCKDB_API virtual unique_ptr<LogStorageScanState> CreateScanState(LoggingTargetTable table) const;
+	DUCKDB_API virtual bool Scan(LogStorageScanState &state, DataChunk &result) const;
+	DUCKDB_API virtual void InitializeScan(LogStorageScanState &state) const;
 
-	DUCKDB_API virtual void Truncate();
+	// Reading interface 2: using bind_replace
+	DUCKDB_API virtual unique_ptr<TableRef> BindReplace(ClientContext &context, TableFunctionBindInput &input,
+	                                                    LoggingTargetTable table);
+
+	//! CONFIGURATION
+	DUCKDB_API virtual void UpdateConfig(DatabaseInstance &db, case_insensitive_map_t<Value> &config);
 };
 
-class StdOutLogStorage : public LogStorage {
+//! The buffering Log storage implements a buffering mechanism around the Base LogStorage class. It implements some
+//! general features that most log storages will need.
+class BufferingLogStorage : public LogStorage {
 public:
-	explicit StdOutLogStorage();
-	~StdOutLogStorage() override;
+	explicit BufferingLogStorage(DatabaseInstance &db_p, idx_t buffer_size, bool normalize);
+	~BufferingLogStorage() override;
+
+	/// (Partially) Implements  LogStorage API
 
-	//! LogStorage API: WRITING
+	//! Write out the entry to the buffers
 	void WriteLogEntry(timestamp_t timestamp, LogLevel level, const string &log_type, const string &log_message,
-	                   const RegisteredLoggingContext &context) override;
-	void WriteLogEntries(DataChunk &chunk, const RegisteredLoggingContext &context) override;
-	void Flush() override;
+	                   const RegisteredLoggingContext &context) final;
+	//! Write out the chunk to the buffers
+	void WriteLogEntries(DataChunk &chunk, const RegisteredLoggingContext &context) final;
+	//! Flushes buffers for all tables
+	void FlushAll() final;
+	//! Flushes buffer for a specific table
+	void Flush(LoggingTargetTable table) final;
+	//! Truncates log storage: both buffers and persistent storage (if applicable)
+	void Truncate() override;
+	//! Apply a new log storage configuration
+	void UpdateConfig(DatabaseInstance &db, case_insensitive_map_t<Value> &config) override;
+	//! Returns whether the table is enabled for this storage
+	bool IsEnabled(LoggingTargetTable table) override;
+
+protected:
+	/// Interface to child classes
+
+	//! Invoked whenever buffers are full to flush to storage
+	virtual void FlushChunk(LoggingTargetTable table, DataChunk &chunk) = 0;
+	//! This method is called in a chained way down the class hierarchy. This allows each class to interpret its own
+	//! part of the config. Unhandled config values that are left over will result in an error
+	virtual void UpdateConfigInternal(DatabaseInstance &db, case_insensitive_map_t<Value> &config);
+	//! ResetAllBuffers will clear all buffered data. To be overridden by child classes to ensure their buffers are
+	//! flushed too
+	virtual void ResetAllBuffers();
+
+	/// Helper methods
+
+	//! Flushes all tables
+	void FlushAllInternal();
+	//! Flushes one of the tables
+	void FlushInternal(LoggingTargetTable table);
+	//! Whether a specific table is available in the log storage
+	bool IsEnabledInternal(LoggingTargetTable table);
+
+	idx_t GetBufferLimit() const;
+
+	//! lock to be used by this class and child classes to ensure thread safety TODO: maybe remove and delegate
+	//! thread-safety to LogManager?
+	mutable mutex lock;
+	//! Switches between using false = use LoggingTargetTable::ALL_LOGS, true = use LoggingTargetTable::LOG_ENTIRES +
+	//! LoggingTargetTable::CONTEXTS
+	bool normalize_contexts = true;
+
+private:
+	//! Resets the log buffers
+	void ResetLogBuffers();
+	//! Write out a logging context
+	void WriteLoggingContext(const RegisteredLoggingContext &context);
+
+	//! The currently registered RegisteredLoggingContext's
+	unordered_set<idx_t> registered_contexts;
+	//! Configuration for buffering
+	idx_t buffer_limit = 0;
+	//! Debug option for testing buffering behaviour
+	bool only_flush_on_full_buffer = false;
+	//! The buffers used for each table
+	unordered_map<LoggingTargetTable, unique_ptr<DataChunk>> buffers;
+	//! This flag is set whenever a new context_is written to the entry buffer. It means that the next flush of
+	//! LoggingTargetTable::LOG_ENTRIES also requires a flush of LoggingTargetTable::LOG_CONTEXTS
+	bool flush_contexts_on_next_entry_flush = false;
+};
+
+//! The CSVLogStorage implements an additional layer on the BufferingLogStorage which will handle converting the log
+//! entries and contexts to CSV lines. It provides functionality to write log data in CSV format with automatic type
+//! casting and configuration of CSV writers. This class serves as a base for both file-based and stdout-based CSV
+//! logging.
+class CSVLogStorage : public BufferingLogStorage {
+public:
+	explicit CSVLogStorage(DatabaseInstance &db, bool normalize, idx_t buffer_size);
+	~CSVLogStorage() override;
+
+protected:
+	/// Implement the BufferingLogStorage interface
+
+	//! Flushes the Chunk to the CSV writers
+	void FlushChunk(LoggingTargetTable table, DataChunk &chunk) final;
+	//! Resets all buffers and state
+	void ResetAllBuffers() override;
+	//! Implements CSVLogStorage specific config handling
+	void UpdateConfigInternal(DatabaseInstance &db, case_insensitive_map_t<Value> &config) override;
+
+	/// Interface to child classes
+
+	//! Hooks for child class to run code pre-flush
+	virtual void BeforeFlush(LoggingTargetTable table, DataChunk &chunk) {};
+	virtual void AfterFlush(LoggingTargetTable table, DataChunk &chunk) {};
+
+	/// Helper functions
+
+	//! To be called by child classed to register the CSV writers used to write to.
+	void RegisterWriter(LoggingTargetTable table, unique_ptr<CSVWriter> writer);
+	//! Returns the writer for a table
+	CSVWriter &GetWriter(LoggingTargetTable table);
+	//! Configure a CSV writer by initializing its settings with the `writer_options` and `reader_options` settings
+	void SetWriterConfigs(CSVWriter &Writer, vector<string> column_names);
+	//! Allows child classes to manipulate options
+	CSVWriterOptions &GetCSVWriterOptions();
+	//! Allows child classes to manipulate options
+	CSVReaderOptions &GetCSVReaderOptions();
+
+private:
+	//! Perform the cast (does not reset input chunk!)
+	void ExecuteCast(LoggingTargetTable table, DataChunk &chunk);
+	//! Reset the Cast chunks
+	void ResetCastChunk();
+	//! Initialize the cast chunks
+	void InitializeCastChunk(LoggingTargetTable table);
+
+	//! The cast buffers used to cast from the original types to the VARCHAR types ready to write to CSV format
+	unordered_map<LoggingTargetTable, unique_ptr<DataChunk>> cast_buffers;
+	//! The writers to be registered by child classes
+	unordered_map<LoggingTargetTable, unique_ptr<CSVWriter>> writers;
 
+	//! CSV Options to initialize the CSVWriters with. TODO: cleanup, this is now a little bit of a mixed bag of
+	//! settings
+	unique_ptr<CSVWriterOptions> writer_options;
+	unique_ptr<CSVReaderOptions> reader_options;
+};
+
+//! Implements a stdout-based log storage using log lines written in CSV format to allow for easy parsing of the log
+//! messages Note that this only supports denormalized logging since there is only 1 practical output stream.
+class StdOutLogStorage : public CSVLogStorage {
+public:
+	explicit StdOutLogStorage(DatabaseInstance &db);
+	~StdOutLogStorage() override;
+
+	const string GetStorageName() override {
+		return "StdOutLogStorage";
+	}
+
+private:
+	class StdOutWriteStream : public WriteStream {
+		void WriteData(const_data_ptr_t buffer, idx_t write_size) override;
+	};
+
+	StdOutWriteStream stdout_stream;
+};
+
+//! FileLogStorage implements a file-based logging system in CSV format.
+//! It implements CSVLogStorage to provide persistent log storage in CSV files. The FileLogStorage can operate in
+//! normalized (separate files for entries and contexts) or denormalized mode (single file)
+class FileLogStorage : public CSVLogStorage {
+public:
+	explicit FileLogStorage(DatabaseInstance &db);
+	~FileLogStorage() override;
+
+	const string GetStorageName() override {
+		return "FileLogStorage";
+	}
+
+	/// Implement LogStorage interface
+
+	//! Truncates the csv files
 	void Truncate() override;
+	//! Bind replace function to scan the different tables
+	unique_ptr<TableRef> BindReplace(ClientContext &context, TableFunctionBindInput &input,
+	                                 LoggingTargetTable table) override;
+
+protected:
+	/// Implement CSVLogStorage interface
+
+	//! Handles the config related to the FileLogStorage
+	void UpdateConfigInternal(DatabaseInstance &db, case_insensitive_map_t<Value> &config) override;
+	//! Lazily initializes the CSV files before first flush
+	void BeforeFlush(LoggingTargetTable table, DataChunk &chunk) override;
+	//! Calls `Sync` on the FileWriters to ensure a LogStorage Flush is flushed to disk immediately
+	void AfterFlush(LoggingTargetTable table, DataChunk &chunk) override;
+
+private:
+	//! Intialize the csv file for `table`
+	void InitializeFile(DatabaseInstance &db, LoggingTargetTable table);
+	//! Initialize the filewriter to be passed to the CSVWriter
+	static unique_ptr<BufferedFileWriter> InitializeFileWriter(DatabaseInstance &db, const string &path);
+	//! Ensures the table is initialized, used in lazy initialization. If already initialized this will NOP
+	void Initialize(LoggingTargetTable table);
+	//! Internal helper function to handle the BindReplace generation
+	unique_ptr<TableRef> BindReplaceInternal(ClientContext &context, TableFunctionBindInput &input, const string &path,
+	                                         const string &select_clause, const string &csv_columns);
+
+	//! DB reference to get the DB filesystem
+	DatabaseInstance &db;
+
+	//! Writer for a table
+	struct TableWriter {
+		//! Passed as WriteStreams to the CSVWriter in the base class
+		unique_ptr<BufferedFileWriter> file_writer;
+		//! Path to initialize the file_writer from
+		string path;
+		//! Whether the file_writer has been (lazily) initialized
+		bool initialized = false;
+	};
+
+	//! The table info per table
+	unordered_map<LoggingTargetTable, TableWriter> tables;
+
+	//! Base path to generate the file paths from
+	string base_path;
+
+private:
+	void SetPaths(const string &base_path);
 };
 
+//! State for scanning the in memory buffers
 class InMemoryLogStorageScanState : public LogStorageScanState {
 public:
-	InMemoryLogStorageScanState();
+	explicit InMemoryLogStorageScanState(LoggingTargetTable table);
 	~InMemoryLogStorageScanState() override;
 
 	ColumnDataScanState scan_state;
 };
 
-class InMemoryLogStorage : public LogStorage {
+//! The InMemoryLogStorage implements a log storage that is backed by ColumnDataCollection's. It only support normalized
+//! mode and support a basic single-threaded scan. TODO: improve?
+class InMemoryLogStorage : public BufferingLogStorage {
 public:
 	explicit InMemoryLogStorage(DatabaseInstance &db);
 	~InMemoryLogStorage() override;
 
-	//! LogStorage API: WRITING
-	void WriteLogEntry(timestamp_t timestamp, LogLevel level, const string &log_type, const string &log_message,
-	                   const RegisteredLoggingContext &context) override;
-	void WriteLogEntries(DataChunk &chunk, const RegisteredLoggingContext &context) override;
-	void Flush() override;
-
-	void Truncate() override;
-
-	//! LogStorage API: READING
-	bool CanScan() override;
-
-	unique_ptr<LogStorageScanState> CreateScanEntriesState() const override;
-	bool ScanEntries(LogStorageScanState &state, DataChunk &result) const override;
-	void InitializeScanEntries(LogStorageScanState &state) const override;
-	unique_ptr<LogStorageScanState> CreateScanContextsState() const override;
-	bool ScanContexts(LogStorageScanState &state, DataChunk &result) const override;
-	void InitializeScanContexts(LogStorageScanState &state) const override;
+	const string GetStorageName() override {
+		return "InMemoryLogStorage";
+	}
 
-protected:
-	void WriteLoggingContext(const RegisteredLoggingContext &context);
-	void ResetBuffers();
+	//! Implement LogStorage Single-threaded scan interface
+	bool CanScan(LoggingTargetTable table) override;
+	unique_ptr<LogStorageScanState> CreateScanState(LoggingTargetTable table) const override;
+	bool Scan(LogStorageScanState &state, DataChunk &result) const override;
+	void InitializeScan(LogStorageScanState &state) const override;
 
 protected:
-	mutable mutex lock;
+	/// Implement BufferingLogStorage interface
 
-	void FlushInternal();
+	//! Flushes a chunk to the corresponding ColumnDataCollection
+	void FlushChunk(LoggingTargetTable table, DataChunk &chunk) override;
+	//! Resets the all ColumnDataCollection's
+	void ResetAllBuffers() override;
 
-	//! Internal log entry storage
-	unique_ptr<ColumnDataCollection> log_entries;
-	unique_ptr<ColumnDataCollection> log_contexts;
-
-	unordered_set<idx_t> registered_contexts;
+private:
+	//! Helper function to get the buffer
+	ColumnDataCollection &GetBuffer(LoggingTargetTable table) const;
 
-	// Cache for direct logging
-	unique_ptr<DataChunk> entry_buffer;
-	unique_ptr<DataChunk> log_context_buffer;
-	idx_t max_buffer_size;
+	unordered_map<LoggingTargetTable, unique_ptr<ColumnDataCollection>> log_storage_buffers;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/appender.hpp

@@ -90,12 +90,14 @@ public:
 	}
 	DUCKDB_API void AppendDataChunk(DataChunk &value);
 
+	virtual void AppendDefault();
+	virtual void AppendDefault(DataChunk &chunk, idx_t col, idx_t row);
 	//! Appends a column to the active column list.
 	//! Immediately flushes all previous data.
-	virtual void AddColumn(const string &name) = 0;
+	virtual void AddColumn(const string &name);
 	//! Removes all columns from the active column list.
 	//! Immediately flushes all previous data.
-	virtual void ClearColumns() = 0;
+	virtual void ClearColumns();
 
 protected:
 	void Destructor();
@@ -125,8 +127,22 @@ protected:
 };
 
 class Appender : public BaseAppender {
+public:
+	DUCKDB_API Appender(Connection &con, const string &database_name, const string &schema_name,
+	                    const string &table_name);
+	DUCKDB_API Appender(Connection &con, const string &schema_name, const string &table_name);
+	DUCKDB_API Appender(Connection &con, const string &table_name);
+	DUCKDB_API ~Appender() override;
+
+public:
+	void AppendDefault() override;
+	void AppendDefault(DataChunk &chunk, idx_t col, idx_t row) override;
+	void AddColumn(const string &name) override;
+	void ClearColumns() override;
+
+private:
 	//! A shared pointer to the context of this appender.
-	shared_ptr<ClientContext> context;
+	weak_ptr<ClientContext> context;
 	//! The table description including the column names.
 	unique_ptr<TableDescription> description;
 	//! All table default values.
@@ -136,22 +152,29 @@ class Appender : public BaseAppender {
 	//! Any other columns default to NULL, or their default values.
 	vector<LogicalIndex> column_ids;
 
-public:
-	DUCKDB_API Appender(Connection &con, const string &database_name, const string &schema_name,
-	                    const string &table_name);
-	DUCKDB_API Appender(Connection &con, const string &schema_name, const string &table_name);
-	DUCKDB_API Appender(Connection &con, const string &table_name);
-	DUCKDB_API ~Appender() override;
+protected:
+	void FlushInternal(ColumnDataCollection &collection) override;
+	Value GetDefaultValue(idx_t column);
+};
 
+class QueryAppender : public BaseAppender {
 public:
-	void AppendDefault();
-	void AppendDefault(DataChunk &chunk, idx_t col, idx_t row);
-	void AddColumn(const string &name) override;
-	void ClearColumns() override;
+	DUCKDB_API QueryAppender(Connection &con, string query, vector<LogicalType> types,
+	                         vector<string> names = vector<string>(), string table_name = string());
+	DUCKDB_API ~QueryAppender() override;
+
+private:
+	//! A shared pointer to the context of this appender.
+	weak_ptr<ClientContext> context;
+	//! The query to run
+	string query;
+	//! The column names of the to-be-appended data, or "col1, col2, ..." if empty
+	vector<string> names;
+	//! The table name that we can reference in the query, or "appended_data" if empty
+	string table_name;
 
 protected:
 	void FlushInternal(ColumnDataCollection &collection) override;
-	Value GetDefaultValue(idx_t column);
 };
 
 class InternalAppender : public BaseAppender {
@@ -167,8 +190,6 @@ public:
 
 protected:
 	void FlushInternal(ColumnDataCollection &collection) override;
-	void AddColumn(const string &name) override;
-	void ClearColumns() override;
 };
 
 template <>