duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/logging/log_storage.cpp

@@ -1,239 +1,783 @@
 #include "duckdb/logging/log_storage.hpp"
+
+#include "duckdb/common/csv_writer.hpp"
+#include "duckdb/common/local_file_system.hpp"
+#include "duckdb/function/table/read_csv.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
+#include "duckdb/main/database_file_opener.hpp"
 #include "duckdb/logging/logging.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/main/client_context.hpp"
-
+#include "duckdb/parser/parser.hpp"
+#include "duckdb/parser/tableref.hpp"
+#include "duckdb/parser/tableref/subqueryref.hpp"
+#include "duckdb/function/cast/vector_cast_helpers.hpp"
+#include "duckdb/common/operator/string_cast.hpp"
+#include "duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp"
+
+#include <complex>
 #include <iostream>
 
 namespace duckdb {
 
-unique_ptr<LogStorageScanState> LogStorage::CreateScanEntriesState() const {
+vector<LogicalType> LogStorage::GetSchema(LoggingTargetTable table) {
+	switch (table) {
+	case LoggingTargetTable::ALL_LOGS:
+		return {
+		    LogicalType::UBIGINT,   // context_id
+		    LogicalType::VARCHAR,   // scope
+		    LogicalType::UBIGINT,   // connection_id
+		    LogicalType::UBIGINT,   // transaction_id
+		    LogicalType::UBIGINT,   // query_id
+		    LogicalType::UBIGINT,   // thread
+		    LogicalType::TIMESTAMP, // timestamp
+		    LogicalType::VARCHAR,   // log_type
+		    LogicalType::VARCHAR,   // level
+		    LogicalType::VARCHAR,   // message
+		};
+	case LoggingTargetTable::LOG_ENTRIES:
+		return {
+		    LogicalType::UBIGINT,   // context_id
+		    LogicalType::TIMESTAMP, // timestamp
+		    LogicalType::VARCHAR,   // log_type
+		    LogicalType::VARCHAR,   // level
+		    LogicalType::VARCHAR,   // message
+		};
+	case LoggingTargetTable::LOG_CONTEXTS:
+		return {
+		    LogicalType::UBIGINT, // context_id
+		    LogicalType::VARCHAR, // scope
+		    LogicalType::UBIGINT, // connection_id
+		    LogicalType::UBIGINT, // transaction_id
+		    LogicalType::UBIGINT, // query_id
+		    LogicalType::UBIGINT, // thread
+		};
+	default:
+		throw NotImplementedException("Unknown logging target table");
+	}
+}
+
+vector<string> LogStorage::GetColumnNames(LoggingTargetTable table) {
+	switch (table) {
+	case LoggingTargetTable::ALL_LOGS:
+		return {
+		    "context_id", "scope",     "connection_id", "transaction_id", "query_id",
+		    "thread_id",  "timestamp", "type",          "log_level",      "message",
+		};
+	case LoggingTargetTable::LOG_ENTRIES:
+		return {"context_id", "timestamp", "type", "log_level", "message"};
+	case LoggingTargetTable::LOG_CONTEXTS:
+		return {
+		    "context_id", "scope", "connection_id", "transaction_id", "query_id", "thread_id",
+		};
+	default:
+		throw NotImplementedException("Unknown logging target table");
+	}
+}
+
+unique_ptr<LogStorageScanState> LogStorage::CreateScanState(LoggingTargetTable table) const {
 	throw NotImplementedException("Not implemented for this LogStorage: CreateScanEntriesState");
 }
-bool LogStorage::ScanEntries(LogStorageScanState &state, DataChunk &result) const {
+bool LogStorage::Scan(LogStorageScanState &state, DataChunk &result) const {
 	throw NotImplementedException("Not implemented for this LogStorage: ScanEntries");
 }
-void LogStorage::InitializeScanEntries(LogStorageScanState &state) const {
+void LogStorage::InitializeScan(LogStorageScanState &state) const {
 	throw NotImplementedException("Not implemented for this LogStorage: InitializeScanEntries");
 }
-unique_ptr<LogStorageScanState> LogStorage::CreateScanContextsState() const {
-	throw NotImplementedException("Not implemented for this LogStorage: CreateScanContextsState");
+void LogStorage::Truncate() {
+	throw NotImplementedException("Not implemented for this LogStorage: TruncateLogStorage");
 }
-bool LogStorage::ScanContexts(LogStorageScanState &state, DataChunk &result) const {
-	throw NotImplementedException("Not implemented for this LogStorage: ScanContexts");
+
+void LogStorage::UpdateConfig(DatabaseInstance &db, case_insensitive_map_t<Value> &config) {
+	if (config.size() > 1) {
+		throw InvalidInputException("LogStorage does not support passing configuration");
+	}
 }
-void LogStorage::InitializeScanContexts(LogStorageScanState &state) const {
-	throw NotImplementedException("Not implemented for this LogStorage: InitializeScanContexts");
+
+unique_ptr<TableRef> LogStorage::BindReplace(ClientContext &context, TableFunctionBindInput &input,
+                                             LoggingTargetTable table) {
+	return nullptr;
 }
-void LogStorage::Truncate() {
-	throw NotImplementedException("Not implemented for this LogStorage: TruncateLogStorage");
+
+CSVLogStorage::~CSVLogStorage() {
+}
+
+CSVLogStorage::CSVLogStorage(DatabaseInstance &db, bool normalize, idx_t buffer_size)
+    : BufferingLogStorage(db, buffer_size, normalize) {
+	reader_options = make_uniq<CSVReaderOptions>();
+	writer_options = make_uniq<CSVWriterOptions>(*reader_options);
+
+	reader_options->dialect_options.state_machine_options.escape = '\"';
+	reader_options->dialect_options.state_machine_options.quote = '\"';
+	reader_options->dialect_options.state_machine_options.delimiter = CSVOption<string>("\t");
+
+	ResetCastChunk();
+}
+
+void BufferingLogStorage::UpdateConfig(DatabaseInstance &db, case_insensitive_map_t<Value> &config) {
+	lock_guard<mutex> lck(lock);
+	return UpdateConfigInternal(db, config);
+}
+
+bool BufferingLogStorage::IsEnabled(LoggingTargetTable table) {
+	lock_guard<mutex> lck(lock);
+	return IsEnabledInternal(table);
+}
+
+bool BufferingLogStorage::IsEnabledInternal(LoggingTargetTable table) {
+	if (normalize_contexts) {
+		return table == LoggingTargetTable::LOG_CONTEXTS || table == LoggingTargetTable::LOG_ENTRIES;
+	}
+
+	return table == LoggingTargetTable::ALL_LOGS;
+}
+
+idx_t BufferingLogStorage::GetBufferLimit() const {
+	return buffer_limit;
+}
+
+void CSVLogStorage::ExecuteCast(LoggingTargetTable table, DataChunk &chunk) {
+	// Reset the cast buffer before use
+	cast_buffers[table]->Reset();
+
+	auto &cast_buffer = *cast_buffers[table];
+	idx_t count = chunk.size();
+
+	// Do default casts
+	for (idx_t i = 0; i < chunk.data.size(); i++) {
+		VectorOperations::DefaultCast(chunk.data[i], cast_buffer.data[i], count, false);
+	}
+	cast_buffer.SetCardinality(count);
+}
+
+void CSVLogStorage::ResetAllBuffers() {
+	BufferingLogStorage::ResetAllBuffers();
+	ResetCastChunk();
+}
+
+void CSVLogStorage::UpdateConfigInternal(DatabaseInstance &db, case_insensitive_map_t<Value> &config) {
+	auto config_copy = config;
+
+	bool changed_writer_settings = false;
+
+	vector<string> to_remove;
+	for (const auto &it : config_copy) {
+		auto key = StringUtil::Lower(it.first);
+		if (key == "delim") {
+			changed_writer_settings = true;
+			reader_options->dialect_options.state_machine_options.delimiter = CSVOption<string>(it.second.ToString());
+			to_remove.push_back(it.first);
+		}
+	}
+
+	if (changed_writer_settings) {
+		for (auto &writer : writers) {
+			SetWriterConfigs(*writer.second, GetColumnNames(writer.first));
+		}
+	}
+
+	for (const auto &it : to_remove) {
+		config_copy.erase(it);
+	}
+
+	return BufferingLogStorage::UpdateConfigInternal(db, config_copy);
+}
+
+void CSVLogStorage::RegisterWriter(LoggingTargetTable table, unique_ptr<CSVWriter> writer) {
+	writers[table] = std::move(writer);
 }
 
-StdOutLogStorage::StdOutLogStorage() {
+CSVWriter &CSVLogStorage::GetWriter(LoggingTargetTable table) {
+	return *writers[table];
+}
+
+void CSVLogStorage::InitializeCastChunk(LoggingTargetTable table) {
+	cast_buffers[table] = make_uniq<DataChunk>();
+
+	vector<LogicalType> types;
+	types.resize(GetSchema(table).size(), LogicalType::VARCHAR);
+	idx_t buffer_size = MaxValue<idx_t>(GetBufferLimit(), 1);
+	cast_buffers[table]->Initialize(Allocator::DefaultAllocator(), types, buffer_size);
+}
+
+void CSVLogStorage::ResetCastChunk() {
+	InitializeCastChunk(LoggingTargetTable::LOG_ENTRIES);
+	InitializeCastChunk(LoggingTargetTable::LOG_CONTEXTS);
+	InitializeCastChunk(LoggingTargetTable::ALL_LOGS);
+}
+
+void CSVLogStorage::SetWriterConfigs(CSVWriter &writer, vector<string> column_names) {
+	writer.options = *reader_options;
+	writer.writer_options = *writer_options;
+
+	// Update the config with the column names since that is different per schema
+	writer.options.name_list = std::move(column_names);
+	writer.options.force_quote = vector<bool>(writer.options.name_list.size(), false);
+}
+
+CSVReaderOptions &CSVLogStorage::GetCSVReaderOptions() {
+	return *reader_options;
+}
+
+CSVWriterOptions &CSVLogStorage::GetCSVWriterOptions() {
+	return *writer_options;
+}
+
+void CSVLogStorage::FlushChunk(LoggingTargetTable table, DataChunk &chunk) {
+	BeforeFlush(table, chunk);
+
+	// Execute the cast
+	ExecuteCast(table, chunk);
+
+	// Write the chunk to the CSVWriter
+	writers[table]->WriteChunk(*cast_buffers[table]);
+	writers[table]->Flush();
+
+	// Call child class to implement any post flushing behaviour (e.g. calling sync)
+	AfterFlush(table, chunk);
+
+	// Reset the cast buffer
+	cast_buffers[table]->Reset();
+}
+
+void BufferingLogStorage::UpdateConfigInternal(DatabaseInstance &db, case_insensitive_map_t<Value> &config) {
+	for (const auto &it : config) {
+		if (StringUtil::Lower(it.first) == "buffer_size") {
+			buffer_limit = it.second.GetValue<uint64_t>();
+			ResetAllBuffers();
+		} else if (StringUtil::Lower(it.first) == "only_flush_on_full_buffer") {
+			// This is a debug option used during testing. It disables the manual
+			only_flush_on_full_buffer = it.second.GetValue<bool>();
+		} else if (StringUtil::Lower(it.first) == "normalize") {
+			throw InternalException("'normalize' setting should be handled in child class");
+		} else {
+			throw InvalidInputException("Unrecognized log storage config option for storage: '%s': '%s'",
+			                            GetStorageName(), it.first);
+		}
+	}
+}
+
+void StdOutLogStorage::StdOutWriteStream::WriteData(const_data_ptr_t buffer, idx_t write_size) {
+	std::cout.write(const_char_ptr_cast(buffer), NumericCast<int64_t>(write_size));
+	std::cout.flush();
+}
+
+StdOutLogStorage::StdOutLogStorage(DatabaseInstance &db) : CSVLogStorage(db, false, 1) {
+	// StdOutLogStorage is denormalized only
+	auto target_table = LoggingTargetTable::ALL_LOGS;
+
+	// Set storage specific defaults
+	GetCSVWriterOptions().newline_writing_mode = CSVNewLineMode::WRITE_AFTER;
+	GetCSVReaderOptions().dialect_options.state_machine_options.delimiter = CSVOption<string>("\t");
+
+	// Create and configure writer
+	auto writer = make_uniq<CSVWriter>(stdout_stream, GetColumnNames(target_table), false);
+	SetWriterConfigs(*writer, GetColumnNames(target_table));
+
+	RegisterWriter(target_table, std::move(writer));
 }
 
 StdOutLogStorage::~StdOutLogStorage() {
 }
 
-void StdOutLogStorage::WriteLogEntry(timestamp_t timestamp, LogLevel level, const string &log_type,
-                                     const string &log_message, const RegisteredLoggingContext &context) {
-	std::cout << StringUtil::Format(
-	    "[LOG] %s, %s, %s, %s, %s, %s, %s, %s\n", Value::TIMESTAMP(timestamp).ToString(), log_type,
-	    EnumUtil::ToString(level), log_message, EnumUtil::ToString(context.context.scope),
-	    context.context.connection_id.IsValid() ? to_string(context.context.connection_id.GetIndex()) : "NULL",
-	    context.context.transaction_id.IsValid() ? to_string(context.context.transaction_id.GetIndex()) : "NULL",
-	    context.context.thread_id.IsValid() ? to_string(context.context.thread_id.GetIndex()) : "NULL");
+FileLogStorage::FileLogStorage(DatabaseInstance &db_p) : CSVLogStorage(db_p, true, STANDARD_VECTOR_SIZE), db(db_p) {
+	tables[LoggingTargetTable::ALL_LOGS] = TableWriter();
+	tables[LoggingTargetTable::LOG_CONTEXTS] = TableWriter();
+	tables[LoggingTargetTable::LOG_ENTRIES] = TableWriter();
+
+	// Set storage specific defaults
+	GetCSVWriterOptions().newline_writing_mode = CSVNewLineMode::WRITE_BEFORE;
+	GetCSVReaderOptions().dialect_options.state_machine_options.delimiter = CSVOption<string>(",");
 }
 
-void StdOutLogStorage::WriteLogEntries(DataChunk &chunk, const RegisteredLoggingContext &context) {
-	throw NotImplementedException("StdOutLogStorage::WriteLogEntries");
+FileLogStorage::~FileLogStorage() {
 }
 
-void StdOutLogStorage::Truncate() {
-	// NOP
+void FileLogStorage::InitializeFile(DatabaseInstance &db, LoggingTargetTable table) {
+	auto &table_writer = tables[table];
+
+	// reset the files writer, we may be re-initializing it here and otherwise we hold 2 handles to the same file
+	table_writer.file_writer.reset();
+
+	// (re)initialize the file writer
+	table_writer.file_writer = InitializeFileWriter(db, table_writer.path);
+	auto file_writer = table_writer.file_writer.get();
+
+	// Create CSV writer that writes to file
+	auto column_names = GetColumnNames(table);
+
+	// Configure writer
+	auto csv_writer = make_uniq<CSVWriter>(*file_writer, column_names, false);
+	SetWriterConfigs(*csv_writer, column_names);
+	bool should_write_header = file_writer->handle->GetFileSize() == 0;
+
+	// We write the header only if the file was empty: when appending to the file we don't
+	csv_writer->options.dialect_options.header = {should_write_header, true};
+
+	// Initialize the writer, this writes out the header if required
+	csv_writer->Initialize();
+
+	// Needed to ensure we correctly start with a newline
+	csv_writer->SetWrittenAnything(true);
+
+	RegisterWriter(table, std::move(csv_writer));
+
+	// Ensures that the file is fully initialized when this function returns
+	file_writer->Sync();
+
+	table_writer.initialized = true;
 }
 
-void StdOutLogStorage::Flush() {
-	// NOP
+unique_ptr<BufferedFileWriter> FileLogStorage::InitializeFileWriter(DatabaseInstance &db, const string &path) {
+	auto &fs = db.GetFileSystem();
+
+	// Create parent directories if non existent
+	auto pos = path.find_last_of(fs.PathSeparator(path));
+	if (pos != path.npos) {
+		fs.CreateDirectoriesRecursive(path.substr(0, pos));
+	}
+
+	FileOpenFlags flags;
+	if (!fs.FileExists(path)) {
+		flags = FileFlags::FILE_FLAGS_DISABLE_LOGGING | FileFlags::FILE_FLAGS_WRITE |
+		        FileFlags::FILE_FLAGS_FILE_CREATE_NEW | FileCompressionType::UNCOMPRESSED;
+	} else {
+		flags = FileFlags::FILE_FLAGS_DISABLE_LOGGING | FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_APPEND;
+	}
+
+	return make_uniq<BufferedFileWriter>(fs, path, flags);
 }
 
-InMemoryLogStorageScanState::InMemoryLogStorageScanState() {
+void FileLogStorage::Truncate() {
+	lock_guard<mutex> lck(lock);
+
+	// Reset buffers
+	ResetAllBuffers();
+
+	for (const auto &it : tables) {
+		auto &file_writer = it.second.file_writer;
+		if (!file_writer) {
+			continue;
+		}
+		// Truncate the file writer
+		file_writer->Truncate(0);
+		// Re-initialize the corresponding CSVWriter
+		GetWriter(it.first).Initialize(true);
+	}
 }
-InMemoryLogStorageScanState::~InMemoryLogStorageScanState() {
+
+void FileLogStorage::BeforeFlush(LoggingTargetTable table, DataChunk &) {
+	// Lazily initialize the files
+	Initialize(table);
 }
 
-InMemoryLogStorage::InMemoryLogStorage(DatabaseInstance &db_p)
-    : entry_buffer(make_uniq<DataChunk>()), log_context_buffer(make_uniq<DataChunk>()) {
-	// LogEntry Schema
-	vector<LogicalType> log_entry_schema = {
-	    LogicalType::UBIGINT,   // context_id
-	    LogicalType::TIMESTAMP, // timestamp
-	    LogicalType::VARCHAR,   // log_type TODO: const vector where possible?
-	    LogicalType::VARCHAR,   // level TODO: enumify
-	    LogicalType::VARCHAR,   // message
-	};
+void FileLogStorage::AfterFlush(LoggingTargetTable table, DataChunk &) {
+	tables[table].file_writer->Sync();
+}
 
-	// LogContext Schema
-	vector<LogicalType> log_context_schema = {
-	    LogicalType::UBIGINT, // context_id
-	    LogicalType::VARCHAR, // scope TODO: enumify
-	    LogicalType::UBIGINT, // connection_id
-	    LogicalType::UBIGINT, // transaction_id
-	    LogicalType::UBIGINT, // query_id
-	    LogicalType::UBIGINT, // thread
-	};
+void FileLogStorage::Initialize(LoggingTargetTable table) {
+	auto &table_writer = tables[table];
+	if (!table_writer.initialized) {
+		if (table_writer.path.empty()) {
+			throw InvalidConfigurationException("Failed to initialize file log storage table, path wasn't set");
+		}
 
-	max_buffer_size = STANDARD_VECTOR_SIZE;
-	entry_buffer->Initialize(Allocator::DefaultAllocator(), log_entry_schema, max_buffer_size);
-	log_context_buffer->Initialize(Allocator::DefaultAllocator(), log_context_schema, max_buffer_size);
-	log_entries = make_uniq<ColumnDataCollection>(db_p.GetBufferManager(), log_entry_schema);
-	log_contexts = make_uniq<ColumnDataCollection>(db_p.GetBufferManager(), log_context_schema);
+		InitializeFile(db, table);
+	}
 }
 
-void InMemoryLogStorage::ResetBuffers() {
-	entry_buffer->Reset();
-	log_context_buffer->Reset();
+void FileLogStorage::SetPaths(const string &base_path) {
+	for (auto &it : tables) {
+		it.second.path.clear();
+	}
+
+	LocalFileSystem fs;
+	if (normalize_contexts) {
+		tables[LoggingTargetTable::LOG_CONTEXTS].path = fs.JoinPath(base_path, "duckdb_log_contexts.csv");
+		tables[LoggingTargetTable::LOG_ENTRIES].path = fs.JoinPath(base_path, "duckdb_log_entries.csv");
+	} else {
+		if (StringUtil::EndsWith(base_path, ".csv")) {
+			tables[LoggingTargetTable::ALL_LOGS].path = base_path;
+		} else {
+			tables[LoggingTargetTable::LOG_ENTRIES].path = fs.JoinPath(base_path, "duckdb_log_entries.csv");
+		}
+	}
+}
 
-	log_entries->Reset();
-	log_contexts->Reset();
+void FileLogStorage::UpdateConfigInternal(DatabaseInstance &db, case_insensitive_map_t<Value> &config) {
+	auto config_copy = config;
+
+	string new_path;
+	bool normalize_contexts_new_value = normalize_contexts;
+	bool normalize_set_explicitly = false;
+	bool require_reinitializing_files = false;
+
+	vector<string> to_remove;
+	for (const auto &it : config_copy) {
+		auto key = StringUtil::Lower(it.first);
+		if (key == "path") {
+			auto path_value = it.second.ToString();
+			//! We implicitly set normalize to false when a path ending in .csv is specified
+			if (!normalize_set_explicitly && StringUtil::EndsWith(path_value, ".csv")) {
+				normalize_contexts_new_value = false;
+			}
+			new_path = path_value;
+			to_remove.push_back(it.first);
+		} else if (key == "normalize") {
+			normalize_set_explicitly = true;
+			normalize_contexts_new_value = it.second.GetValue<bool>();
+			to_remove.push_back(it.first);
+		} else if (key == "delim") {
+			require_reinitializing_files = true;
+		}
+	}
+
+	if (StringUtil::EndsWith(new_path, ".csv") && normalize_contexts_new_value) {
+		throw InvalidConfigurationException(
+		    "Can not set path to '%s' while normalize is true. Normalize will make DuckDB write multiple log files to "
+		    "more efficiently store log entries. Please specify a directory path instead of a csv file path, or set "
+		    "normalize to false.",
+		    new_path);
+	}
+
+	// If any writer is initialized, we flush first:
+	// - when switching between normalized and denormalized, this is necessary since we are changing the buffer schema
+	// - when simply changing the path, it avoids writing log entries written before this change to end up in the new
+	// file
+	bool initialized = false;
+	for (auto &it : tables) {
+		initialized |= it.second.initialized;
+	}
+	if (initialized) {
+		FlushAllInternal();
+	}
 
-	registered_contexts.clear();
+	require_reinitializing_files |= normalize_contexts != normalize_contexts_new_value;
+	normalize_contexts = normalize_contexts_new_value;
+
+	// Reset the buffers to ensure they have the correct schema
+	if (require_reinitializing_files) {
+		ResetAllBuffers();
+
+		// Mark tables as uninitialized
+		for (auto &table : tables) {
+			table.second.initialized = false;
+		}
+	}
+
+	// Apply any path change
+	if (new_path != base_path) {
+		base_path = new_path;
+		SetPaths(new_path);
+	}
+
+	for (const auto &it : to_remove) {
+		config_copy.erase(it);
+	}
+
+	CSVLogStorage::UpdateConfigInternal(db, config_copy);
 }
 
-InMemoryLogStorage::~InMemoryLogStorage() {
+unique_ptr<TableRef> FileLogStorage::BindReplaceInternal(ClientContext &context, TableFunctionBindInput &input,
+                                                         const string &path, const string &select_clause,
+                                                         const string &csv_columns) {
+	string sub_query_string;
+
+	string escaped_path = KeywordHelper::WriteOptionallyQuoted(path);
+	sub_query_string =
+	    StringUtil::Format("%s FROM read_csv_auto(%s, columns={%s})", select_clause, escaped_path, csv_columns);
+
+	Parser parser(context.GetParserOptions());
+	parser.ParseQuery(sub_query_string);
+	auto select_stmt = unique_ptr_cast<SQLStatement, SelectStatement>(std::move(parser.statements[0]));
+
+	return duckdb::make_uniq<SubqueryRef>(std::move(select_stmt));
 }
 
-void InMemoryLogStorage::WriteLogEntry(timestamp_t timestamp, LogLevel level, const string &log_type,
-                                       const string &log_message, const RegisteredLoggingContext &context) {
-	unique_lock<mutex> lck(lock);
+unique_ptr<TableRef> FileLogStorage::BindReplace(ClientContext &context, TableFunctionBindInput &input,
+                                                 LoggingTargetTable table) {
+	lock_guard<mutex> lck(lock);
 
-	if (registered_contexts.find(context.context_id) == registered_contexts.end()) {
-		WriteLoggingContext(context);
+	// We only allow scanning enabled tables
+	if (!IsEnabledInternal(table)) {
+		return nullptr;
 	}
 
-	auto size = entry_buffer->size();
-	auto context_id_data = FlatVector::GetData<idx_t>(entry_buffer->data[0]);
-	auto timestamp_data = FlatVector::GetData<timestamp_t>(entry_buffer->data[1]);
-	auto type_data = FlatVector::GetData<string_t>(entry_buffer->data[2]);
-	auto level_data = FlatVector::GetData<string_t>(entry_buffer->data[3]);
-	auto message_data = FlatVector::GetData<string_t>(entry_buffer->data[4]);
+	// We start by flushing the table to ensure we scan the latest version
+	FlushInternal(table);
 
-	context_id_data[size] = context.context_id;
-	timestamp_data[size] = timestamp;
-	type_data[size] = StringVector::AddString(entry_buffer->data[2], log_type);
-	level_data[size] = StringVector::AddString(entry_buffer->data[3], EnumUtil::ToString(level));
-	message_data[size] = StringVector::AddString(entry_buffer->data[4], log_message);
+	if (normalize_contexts && table == LoggingTargetTable::ALL_LOGS) {
+		throw InvalidConfigurationException("Can not scan ALL_LOGS table when logs are normalized");
+	}
+	if (!normalize_contexts && table != LoggingTargetTable::ALL_LOGS) {
+		throw InvalidConfigurationException("Can only scan ALL_LOGS table when logs are normalized");
+	}
+
+	string select = "SELECT *";
+	string path = tables[table].path;
+
+	string columns;
+	if (table == LoggingTargetTable::LOG_ENTRIES) {
+		columns = "'context_id': 'UBIGINT', 'timestamp': 'TIMESTAMP WITH TIME ZONE', 'type': 'VARCHAR', 'log_level': "
+		          "'VARCHAR' , "
+		          "'message': 'VARCHAR'";
+	} else if (table == LoggingTargetTable::LOG_CONTEXTS) {
+		columns = "'context_id': 'UBIGINT', 'scope': 'VARCHAR', 'connection_id': 'UBIGINT', 'transaction_id': "
+		          "'UBIGINT', 'query_id': 'UBIGINT', 'thread_id': 'UBIGINT'";
+	} else if (table == LoggingTargetTable::ALL_LOGS) {
+		select = "SELECT context_id, scope, connection_id, transaction_id, query_id, thread_id, timestamp, type, "
+		         "log_level, message ";
+		columns = "'context_id': 'UBIGINT', 'scope': 'VARCHAR', 'connection_id': 'UBIGINT', 'transaction_id': "
+		          "'UBIGINT', 'query_id': 'UBIGINT', 'thread_id': 'UBIGINT', 'timestamp': 'TIMESTAMP WITH TIME ZONE', "
+		          "'type': "
+		          "'VARCHAR', 'log_level': 'VARCHAR' , 'message': 'VARCHAR'";
+	} else {
+		throw InternalException("Invalid logging target table");
+	}
+
+	return BindReplaceInternal(context, input, path, select, columns);
+}
 
-	entry_buffer->SetCardinality(size + 1);
+BufferingLogStorage::BufferingLogStorage(DatabaseInstance &db_p, idx_t buffer_size, bool normalize)
+    : normalize_contexts(normalize), buffer_limit(buffer_size) {
+	ResetLogBuffers();
+}
+
+void BufferingLogStorage::ResetLogBuffers() {
+	idx_t buffer_size = MaxValue<idx_t>(buffer_limit, 1);
+	if (normalize_contexts) {
+		buffers[LoggingTargetTable::LOG_ENTRIES] = make_uniq<DataChunk>();
+		buffers[LoggingTargetTable::LOG_CONTEXTS] = make_uniq<DataChunk>();
+		buffers[LoggingTargetTable::LOG_ENTRIES]->Initialize(Allocator::DefaultAllocator(),
+		                                                     GetSchema(LoggingTargetTable::LOG_ENTRIES), buffer_size);
+		buffers[LoggingTargetTable::LOG_CONTEXTS]->Initialize(Allocator::DefaultAllocator(),
+		                                                      GetSchema(LoggingTargetTable::LOG_CONTEXTS), buffer_size);
 
-	if (size + 1 >= max_buffer_size) {
-		FlushInternal();
+	} else {
+		buffers[LoggingTargetTable::ALL_LOGS] = make_uniq<DataChunk>();
+		buffers[LoggingTargetTable::ALL_LOGS]->Initialize(Allocator::DefaultAllocator(),
+		                                                  GetSchema(LoggingTargetTable::ALL_LOGS), buffer_size);
 	}
 }
 
-void InMemoryLogStorage::WriteLogEntries(DataChunk &chunk, const RegisteredLoggingContext &context) {
-	log_entries->Append(chunk);
+void BufferingLogStorage::ResetAllBuffers() {
+	ResetLogBuffers();
 }
 
-void InMemoryLogStorage::Flush() {
-	unique_lock<mutex> lck(lock);
-	FlushInternal();
+InMemoryLogStorageScanState::InMemoryLogStorageScanState(LoggingTargetTable table) : LogStorageScanState(table) {
+}
+InMemoryLogStorageScanState::~InMemoryLogStorageScanState() {
 }
 
-void InMemoryLogStorage::Truncate() {
-	unique_lock<mutex> lck(lock);
-	ResetBuffers();
+InMemoryLogStorage::InMemoryLogStorage(DatabaseInstance &db_p) : BufferingLogStorage(db_p, STANDARD_VECTOR_SIZE, true) {
+	log_storage_buffers[LoggingTargetTable::LOG_ENTRIES] =
+	    make_uniq<ColumnDataCollection>(Allocator::DefaultAllocator(), GetSchema(LoggingTargetTable::LOG_ENTRIES));
+	log_storage_buffers[LoggingTargetTable::LOG_CONTEXTS] =
+	    make_uniq<ColumnDataCollection>(Allocator::DefaultAllocator(), GetSchema(LoggingTargetTable::LOG_CONTEXTS));
 }
 
-void InMemoryLogStorage::FlushInternal() {
-	if (entry_buffer->size() > 0) {
-		log_entries->Append(*entry_buffer);
-		entry_buffer->Reset();
+void InMemoryLogStorage::ResetAllBuffers() {
+	BufferingLogStorage::ResetAllBuffers();
+
+	for (const auto &buffer : log_storage_buffers) {
+		buffer.second->Reset();
 	}
+}
 
-	if (log_context_buffer->size() > 0) {
-		log_contexts->Append(*log_context_buffer);
-		log_context_buffer->Reset();
+ColumnDataCollection &InMemoryLogStorage::GetBuffer(LoggingTargetTable table) const {
+	auto res = log_storage_buffers.find(table);
+	if (res == log_storage_buffers.end()) {
+		throw InternalException("Failed to find table");
 	}
+	return *res->second;
 }
 
-void InMemoryLogStorage::WriteLoggingContext(const RegisteredLoggingContext &context) {
-	registered_contexts.insert(context.context_id);
+InMemoryLogStorage::~InMemoryLogStorage() {
+}
 
-	auto size = log_context_buffer->size();
+BufferingLogStorage::~BufferingLogStorage() {
+}
+
+static void WriteLoggingContextsToChunk(DataChunk &chunk, const RegisteredLoggingContext &context, idx_t &col) {
+
+	auto size = chunk.size();
 
-	auto context_id_data = FlatVector::GetData<idx_t>(log_context_buffer->data[0]);
+	auto context_id_data = FlatVector::GetData<idx_t>(chunk.data[col++]);
 	context_id_data[size] = context.context_id;
 
-	auto context_scope_data = FlatVector::GetData<string_t>(log_context_buffer->data[1]);
-	context_scope_data[size] =
-	    StringVector::AddString(log_context_buffer->data[1], EnumUtil::ToString(context.context.scope));
+	auto context_scope_data = FlatVector::GetData<string_t>(chunk.data[col]);
+	context_scope_data[size] = StringVector::AddString(chunk.data[col++], EnumUtil::ToString(context.context.scope));
 
 	if (context.context.connection_id.IsValid()) {
-		auto client_context_data = FlatVector::GetData<idx_t>(log_context_buffer->data[2]);
+		auto client_context_data = FlatVector::GetData<idx_t>(chunk.data[col++]);
 		client_context_data[size] = context.context.connection_id.GetIndex();
 	} else {
-		FlatVector::Validity(log_context_buffer->data[2]).SetInvalid(size);
+		FlatVector::Validity(chunk.data[col++]).SetInvalid(size);
 	}
 	if (context.context.transaction_id.IsValid()) {
-		auto client_context_data = FlatVector::GetData<idx_t>(log_context_buffer->data[3]);
+		auto client_context_data = FlatVector::GetData<idx_t>(chunk.data[col++]);
 		client_context_data[size] = context.context.transaction_id.GetIndex();
 	} else {
-		FlatVector::Validity(log_context_buffer->data[3]).SetInvalid(size);
+		FlatVector::Validity(chunk.data[col++]).SetInvalid(size);
 	}
 	if (context.context.query_id.IsValid()) {
-		auto client_context_data = FlatVector::GetData<idx_t>(log_context_buffer->data[4]);
+		auto client_context_data = FlatVector::GetData<idx_t>(chunk.data[col++]);
 		client_context_data[size] = context.context.query_id.GetIndex();
 	} else {
-		FlatVector::Validity(log_context_buffer->data[4]).SetInvalid(size);
+		FlatVector::Validity(chunk.data[col++]).SetInvalid(size);
 	}
 
 	if (context.context.thread_id.IsValid()) {
-		auto thread_data = FlatVector::GetData<idx_t>(log_context_buffer->data[5]);
+		auto thread_data = FlatVector::GetData<idx_t>(chunk.data[col++]);
 		thread_data[size] = context.context.thread_id.GetIndex();
 	} else {
-		FlatVector::Validity(log_context_buffer->data[5]).SetInvalid(size);
+		FlatVector::Validity(chunk.data[col++]).SetInvalid(size);
+	}
+
+	chunk.SetCardinality(size + 1);
+}
+
+void BufferingLogStorage::WriteLogEntry(timestamp_t timestamp, LogLevel level, const string &log_type,
+                                        const string &log_message, const RegisteredLoggingContext &context) {
+	unique_lock<mutex> lck(lock);
+
+	auto &log_entries_buffer =
+	    normalize_contexts ? buffers[LoggingTargetTable::LOG_ENTRIES] : buffers[LoggingTargetTable::ALL_LOGS];
+
+	auto size = log_entries_buffer->size();
+	if (size >= buffer_limit && buffer_limit != 0) {
+		throw InternalException("Log buffer limit exceeded: code should have flushed before");
+	}
+
+	if (registered_contexts.find(context.context_id) == registered_contexts.end()) {
+		WriteLoggingContext(context);
+		// New context_id: we should flush both contexts and entries next time LOG_ENTRIES gets flushed
+		flush_contexts_on_next_entry_flush = true;
+	}
+
+	idx_t col = 0;
+
+	if (normalize_contexts) {
+		auto context_id_data = FlatVector::GetData<idx_t>(log_entries_buffer->data[col++]);
+		context_id_data[size] = context.context_id;
+	} else {
+		WriteLoggingContextsToChunk(*log_entries_buffer, context, col);
 	}
 
-	log_context_buffer->SetCardinality(size + 1);
+	auto timestamp_data = FlatVector::GetData<timestamp_t>(log_entries_buffer->data[col++]);
+	timestamp_data[size] = timestamp;
 
-	if (size + 1 >= max_buffer_size) {
-		FlushInternal();
+	auto type_data = FlatVector::GetData<string_t>(log_entries_buffer->data[col]);
+	type_data[size] = StringVector::AddString(log_entries_buffer->data[col++], log_type);
+
+	auto level_data = FlatVector::GetData<string_t>(log_entries_buffer->data[col]);
+	level_data[size] = StringVector::AddString(log_entries_buffer->data[col++],
+	                                           EnumUtil::ToString(level)); // TODO: do cast on write out
+
+	auto message_data = FlatVector::GetData<string_t>(log_entries_buffer->data[col]);
+	message_data[size] = StringVector::AddString(log_entries_buffer->data[col++], log_message);
+
+	log_entries_buffer->SetCardinality(size + 1);
+
+	if (size + 1 >= buffer_limit) {
+		if (normalize_contexts) {
+			// Flush all entries
+			FlushInternal(LoggingTargetTable::LOG_ENTRIES);
+			// Flush contexts if required
+			if (flush_contexts_on_next_entry_flush) {
+				FlushInternal(LoggingTargetTable::LOG_CONTEXTS);
+				flush_contexts_on_next_entry_flush = false;
+			}
+		} else {
+			FlushInternal(LoggingTargetTable::ALL_LOGS);
+		}
 	}
 }
 
-bool InMemoryLogStorage::CanScan() {
-	return true;
+void BufferingLogStorage::WriteLogEntries(DataChunk &chunk, const RegisteredLoggingContext &context) {
+	throw NotImplementedException("BufferingLogStorage::WriteLogEntries(DataChunk &chunk) not implemented");
 }
 
-unique_ptr<LogStorageScanState> InMemoryLogStorage::CreateScanEntriesState() const {
-	return make_uniq<InMemoryLogStorageScanState>();
+void BufferingLogStorage::FlushAll() {
+	unique_lock<mutex> lck(lock);
+	if (!only_flush_on_full_buffer) {
+		FlushAllInternal();
+	}
 }
-bool InMemoryLogStorage::ScanEntries(LogStorageScanState &state, DataChunk &result) const {
+
+void BufferingLogStorage::Flush(LoggingTargetTable table) {
 	unique_lock<mutex> lck(lock);
-	auto &in_mem_scan_state = state.Cast<InMemoryLogStorageScanState>();
-	return log_entries->Scan(in_mem_scan_state.scan_state, result);
+	if (!only_flush_on_full_buffer) {
+		FlushInternal(table);
+	}
 }
 
-void InMemoryLogStorage::InitializeScanEntries(LogStorageScanState &state) const {
+void BufferingLogStorage::Truncate() {
 	unique_lock<mutex> lck(lock);
-	auto &in_mem_scan_state = state.Cast<InMemoryLogStorageScanState>();
-	log_entries->InitializeScan(in_mem_scan_state.scan_state, ColumnDataScanProperties::DISALLOW_ZERO_COPY);
+	ResetAllBuffers();
+}
+
+void InMemoryLogStorage::FlushChunk(LoggingTargetTable table, DataChunk &chunk) {
+	D_ASSERT(table == LoggingTargetTable::LOG_ENTRIES || table == LoggingTargetTable::LOG_CONTEXTS);
+	log_storage_buffers[table]->Append(chunk);
+}
+
+void BufferingLogStorage::FlushAllInternal() {
+	if (normalize_contexts) {
+		FlushInternal(LoggingTargetTable::LOG_ENTRIES);
+		FlushInternal(LoggingTargetTable::LOG_CONTEXTS);
+	} else {
+		FlushInternal(LoggingTargetTable::ALL_LOGS);
+	}
+}
+
+void BufferingLogStorage::FlushInternal(LoggingTargetTable table) {
+	if (!IsEnabledInternal(table)) {
+		throw InvalidConfigurationException("Cannot flush disabled logging target");
+	}
+	FlushChunk(table, *buffers[table]);
+	buffers[table]->Reset();
+}
+
+void BufferingLogStorage::WriteLoggingContext(const RegisteredLoggingContext &context) {
+	registered_contexts.insert(context.context_id);
+
+	// If we don't normalize the contexts they are written out on every log entry
+	if (!normalize_contexts) {
+		return;
+	}
+
+	idx_t col = 0;
+
+	auto &log_contexts_buffer = buffers[LoggingTargetTable::LOG_CONTEXTS];
+
+	if (log_contexts_buffer->size() + 1 > buffer_limit) {
+		FlushInternal(LoggingTargetTable::LOG_CONTEXTS);
+	}
+
+	WriteLoggingContextsToChunk(*log_contexts_buffer, context, col);
+}
+
+bool InMemoryLogStorage::CanScan(LoggingTargetTable table) {
+	unique_lock<mutex> lck(lock);
+	return IsEnabledInternal(table);
 }
 
-unique_ptr<LogStorageScanState> InMemoryLogStorage::CreateScanContextsState() const {
-	return make_uniq<InMemoryLogStorageScanState>();
+unique_ptr<LogStorageScanState> InMemoryLogStorage::CreateScanState(LoggingTargetTable table) const {
+	return make_uniq<InMemoryLogStorageScanState>(table);
 }
-bool InMemoryLogStorage::ScanContexts(LogStorageScanState &state, DataChunk &result) const {
+
+bool InMemoryLogStorage::Scan(LogStorageScanState &state, DataChunk &result) const {
 	unique_lock<mutex> lck(lock);
 	auto &in_mem_scan_state = state.Cast<InMemoryLogStorageScanState>();
-	return log_contexts->Scan(in_mem_scan_state.scan_state, result);
+	return GetBuffer(in_mem_scan_state.table).Scan(in_mem_scan_state.scan_state, result);
 }
 
-void InMemoryLogStorage::InitializeScanContexts(LogStorageScanState &state) const {
+void InMemoryLogStorage::InitializeScan(LogStorageScanState &state) const {
 	unique_lock<mutex> lck(lock);
 	auto &in_mem_scan_state = state.Cast<InMemoryLogStorageScanState>();
-	log_contexts->InitializeScan(in_mem_scan_state.scan_state, ColumnDataScanProperties::DISALLOW_ZERO_COPY);
+	GetBuffer(in_mem_scan_state.table)
+	    .InitializeScan(in_mem_scan_state.scan_state, ColumnDataScanProperties::DISALLOW_ZERO_COPY);
 }
 
 } // namespace duckdb