duckdb 1.3.3-dev0.0 → 1.3.3-dev4.0

package/src/duckdb/src/include/duckdb/common/bignum.hpp

@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/bignum.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/winapi.hpp"
+#include "duckdb/common/string.hpp"
+#include <stdint.h>
+#include "duckdb/common/types/string_type.hpp"
+#include "duckdb/storage/arena_allocator.hpp"
+
+namespace duckdb {
+
+struct bignum_t {
+	string_t data;
+
+	bignum_t() : data() {
+	}
+
+	explicit bignum_t(const string_t &data) : data(data) {
+	}
+
+	bignum_t(const bignum_t &rhs) = default;
+	bignum_t(bignum_t &&other) = default;
+	bignum_t &operator=(const bignum_t &rhs) = default;
+	bignum_t &operator=(bignum_t &&rhs) = default;
+
+	void Print() const;
+};
+
+enum AbsoluteNumberComparison : uint8_t {
+	// If number is equal
+	EQUAL = 0,
+	// If compared number is greater
+	GREATER = 1,
+	// If compared number is smaller
+	SMALLER = 2,
+};
+
+struct BignumIntermediate {
+	BignumIntermediate() : is_negative(false), size(0), data(nullptr) {};
+	explicit BignumIntermediate(const bignum_t &value);
+	BignumIntermediate(uint8_t *value, idx_t size);
+	void Print() const;
+	//! Information on the header
+	bool is_negative;
+	uint32_t size;
+	//! The actual data
+	data_ptr_t data;
+	//! If the absolute number is  bigger than the absolute rhs
+	//! 1 = true, 0 = equal, -1 = false
+	AbsoluteNumberComparison IsAbsoluteBigger(const BignumIntermediate &rhs) const;
+	//! Get the absolute value of a byte
+	uint8_t GetAbsoluteByte(int64_t index) const;
+	//! If the most significant bit of the first byte is set.
+	bool IsMSBSet() const;
+	//! Initializes our bignum to 0 and 1 byte
+	void Initialize(ArenaAllocator &allocator);
+	//! If necessary, we reallocate our intermediate to the next power of 2.
+	void Reallocate(ArenaAllocator &allocator, idx_t min_size);
+	static uint32_t GetStartDataPos(data_ptr_t data, idx_t size, bool is_negative);
+	uint32_t GetStartDataPos() const;
+	//! In case we have unnecessary extra 0's or 1's in our bignum we trim them
+	static idx_t Trim(data_ptr_t data, uint32_t &size, bool is_negative);
+	void Trim();
+	//! Add a BignumIntermediate to another BignumIntermediate, equivalent of a +=
+	void AddInPlace(ArenaAllocator &allocator, const BignumIntermediate &rhs);
+	//! Adds two BignumIntermediates and returns a string_t result, equivalent of a +
+	static string_t Add(Vector &result, const BignumIntermediate &lhs, const BignumIntermediate &rhs);
+	//! Negates a value, e.g., -x
+	string_t Negate(Vector &result_vector) const;
+	void NegateInPlace();
+	//! Exports to a bignum, either arena allocated
+	bignum_t ToBignum(ArenaAllocator &allocator);
+	//! Check if an over/underflow has occurred
+	static bool OverOrUnderflow(data_ptr_t data, idx_t size, bool is_negative);
+	bool OverOrUnderflow() const;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/box_renderer.hpp

@@ -164,10 +164,10 @@ private:
 	                  idx_t total_length, bool has_results, BaseResultRenderer &renderer);
 	void RenderValues(const list<ColumnDataCollection> &collections, const vector<idx_t> &column_map,
 	                  const vector<idx_t> &widths, const vector<LogicalType> &result_types, BaseResultRenderer &ss);
-	void RenderRowCount(string row_count_str, string shown_str, const string &column_count_str,
-	                    const vector<idx_t> &boundaries, bool has_hidden_rows, bool has_hidden_columns,
-	                    idx_t total_length, idx_t row_count, idx_t column_count, idx_t minimum_row_length,
-	                    BaseResultRenderer &ss);
+	void RenderRowCount(string &row_count_str, string &readable_rows_str, string &shown_str,
+	                    const string &column_count_str, const vector<idx_t> &boundaries, bool has_hidden_rows,
+	                    bool has_hidden_columns, idx_t total_length, idx_t row_count, idx_t column_count,
+	                    idx_t minimum_row_length, BaseResultRenderer &ss);
 
 	string FormatNumber(const string &input);
 	string ConvertRenderValue(const string &input, const LogicalType &type);

package/src/duckdb/src/include/duckdb/common/chrono.hpp

@@ -14,6 +14,7 @@ namespace duckdb {
 using std::chrono::duration;
 using std::chrono::duration_cast;
 using std::chrono::high_resolution_clock;
+using std::chrono::microseconds;
 using std::chrono::milliseconds;
 using std::chrono::nanoseconds;
 using std::chrono::steady_clock;

package/src/duckdb/src/include/duckdb/common/complex_json.hpp

@@ -11,22 +11,31 @@
 #include <unordered_map>
 #include <string>
 #include "duckdb/common/exception.hpp"
+#include "duckdb/common/vector.hpp"
 #include "duckdb/common/unique_ptr.hpp"
 
 namespace duckdb {
 
+enum class ComplexJSONType : uint8_t { VALUE = 0, OBJECT = 1, ARRAY = 2 };
+
 //! Custom struct to handle both strings and nested JSON objects
 struct ComplexJSON {
 	//! Constructor for string values
 	explicit ComplexJSON(const string &str);
 	//! Basic empty constructor
 	ComplexJSON();
-	//! Adds Object to the underlying map, also sets the is_object flag to True
-	void AddObject(const string &key, unique_ptr<ComplexJSON> object);
+	//! Adds entry to the underlying map, also sets the type to OBJECT
+	void AddObjectEntry(const string &key, unique_ptr<ComplexJSON> object);
+	//! Adds element to the underlying list, also sets the type to ARRAY
+	void AddArrayElement(unique_ptr<ComplexJSON> object);
 	//! Gets a ComplexJSON object from the map
 	ComplexJSON &GetObject(const string &key);
+	//! Gets a ComplexJSON element from the list
+	ComplexJSON &GetArrayElement(const idx_t &index);
 	//! Gets a string version of the underlying ComplexJSON object from the map
 	string GetValue(const string &key) const;
+	//! Gets a string version of the underlying ComplexJSON array from the list
+	string GetValue(const idx_t &index) const;
 	//! Recursive function for GetValue
 	static string GetValueRecursive(const ComplexJSON &child);
 	//! Flattens this json to a top level key -> nested json
@@ -35,10 +44,12 @@ struct ComplexJSON {
 private:
 	//! Basic string value, in case this is the last value of a nested json
 	string str_value;
-	//! If this is a complex json a map of key/value
+	//! If this is a json object a map of key/value
 	unordered_map<string, unique_ptr<ComplexJSON>> obj_value;
+	//! If this is a json array a list of values
+	vector<unique_ptr<ComplexJSON>> arr_value;
 	//! If this json is an object (i.e., map or not)
-	bool is_object;
+	ComplexJSONType type;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/compressed_file_system.hpp

@@ -32,7 +32,7 @@ struct StreamData {
 struct StreamWrapper {
 	DUCKDB_API virtual ~StreamWrapper();
 
-	DUCKDB_API virtual void Initialize(CompressedFile &file, bool write) = 0;
+	DUCKDB_API virtual void Initialize(QueryContext context, CompressedFile &file, bool write) = 0;
 	DUCKDB_API virtual bool Read(StreamData &stream_data) = 0;
 	DUCKDB_API virtual void Write(CompressedFile &file, StreamData &stream_data, data_ptr_t buffer,
 	                              int64_t nr_bytes) = 0;
@@ -70,7 +70,7 @@ public:
 	StreamData stream_data;
 
 public:
-	DUCKDB_API void Initialize(bool write);
+	DUCKDB_API void Initialize(QueryContext context, bool write);
 	DUCKDB_API int64_t ReadData(void *buffer, int64_t nr_bytes);
 	DUCKDB_API int64_t WriteData(data_ptr_t buffer, int64_t nr_bytes);
 	DUCKDB_API void Close() override;

package/src/duckdb/src/include/duckdb/common/csv_writer.hpp

@@ -0,0 +1,153 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/csv_writer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
+
+namespace duckdb {
+class MemoryStream;
+
+enum class CSVNewLineMode {
+	//! Newlines are written before writing out a new csv line. Ensures we don't write an empty line at the end of
+	//! a csv file. Uses CSVWriterState::written_anything to keep track of when to insert newlines
+	WRITE_BEFORE = 0,
+	//! Newlines are written after every line. This is cleanest in stdout, where lines are expected to end with a
+	//! newline
+	WRITE_AFTER = 1
+};
+
+struct CSVWriterOptions {
+	CSVWriterOptions(const string &delim, const char &quote, const string &write_newline);
+	explicit CSVWriterOptions(CSVReaderOptions &options);
+
+	//! The newline string to write
+	string newline = "\n";
+	//! The size of the CSV file (in bytes) that we buffer before we flush it to disk
+	idx_t flush_size = 4096ULL * 8ULL;
+	//! For each byte whether the CSV file requires quotes when containing the byte
+	vector<bool> requires_quotes;
+	//! How to write newlines
+	CSVNewLineMode newline_writing_mode = CSVNewLineMode::WRITE_BEFORE;
+};
+
+struct CSVWriterState {
+	CSVWriterState(ClientContext &context, idx_t flush_size);
+	CSVWriterState(DatabaseInstance &db, idx_t flush_size);
+	CSVWriterState();
+	~CSVWriterState();
+
+	void Reset() {
+		stream->Rewind();
+		written_anything = false;
+	}
+
+	idx_t flush_size;
+	unique_ptr<MemoryStream> stream;
+	bool written_anything = false;
+
+	bool require_manual_flush = false;
+};
+
+class CSVWriter {
+public:
+	//! Create a CSVWriter that writes to a (non-owned) WriteStream
+	CSVWriter(WriteStream &stream, vector<string> name_list, bool shared = true);
+
+	//! Create a CSVWriter that writes to a file
+	CSVWriter(CSVReaderOptions &options, FileSystem &fs, const string &file_path, FileCompressionType compression,
+	          bool shared = true);
+
+	//! Writes header and prefix if necessary
+	void Initialize(bool force = false);
+
+	//! Writes the raw string directly into the output stream
+	void WriteRawString(const string &data);
+	//! Writes the header directly into the output stream
+	void WriteHeader();
+	//! Write the Raw String, using the local_state
+	void WriteRawString(const string &prefix, CSVWriterState &local_state);
+	//! Write a chunk of VARCHAR vectors to the CSV file (any casts are the responsibility of caller)
+	void WriteChunk(DataChunk &input, CSVWriterState &local_state);
+	//! (Non-shared only) variant of WriteChunk
+	void WriteChunk(DataChunk &input);
+
+	//! Flushes all data in the local write state
+	void Flush(CSVWriterState &local_state);
+	//! (Non-shared only) variant of Flush
+	void Flush();
+
+	//! Resets the state of the writer. Warning: the file_writer is not reset
+	void Reset(optional_ptr<CSVWriterState> local_state);
+
+	//! Closes the writer, optionally writes a postfix
+	void Close();
+
+	unique_ptr<CSVWriterState> InitializeLocalWriteState(ClientContext &context, idx_t flush_size);
+	unique_ptr<CSVWriterState> InitializeLocalWriteState(DatabaseInstance &db, idx_t flush_size);
+
+	vector<unique_ptr<Expression>> string_casts;
+
+	idx_t BytesWritten();
+
+	//! BytesWritten + OriginalSize;
+	idx_t FileSize();
+
+	bool WrittenAnything() {
+		return written_anything;
+	}
+	void SetWrittenAnything(bool val) {
+		if (shared) {
+			lock_guard<mutex> guard(lock);
+			written_anything = val;
+		} else {
+			written_anything = val;
+		}
+	}
+
+	CSVReaderOptions options;
+	CSVWriterOptions writer_options;
+
+protected:
+	void FlushInternal(CSVWriterState &local_state);
+	void ResetInternal(optional_ptr<CSVWriterState> local_state);
+
+	//! If we've written any rows yet, allows us to prevent a trailing comma when writing JSON ARRAY
+	bool written_anything = false;
+
+	//! (optional) The owned file writer of this CSVWriter
+	unique_ptr<BufferedFileWriter> file_writer;
+
+	//! The WriteStream to write the CSV data to
+	WriteStream &write_stream;
+
+	idx_t bytes_written = 0;
+
+	bool should_initialize;
+
+	mutex lock;
+	bool shared;
+
+	unique_ptr<CSVWriterState> global_write_state;
+
+	static void WriteQuoteOrEscape(WriteStream &writer, char quote_or_escape);
+	static string AddEscapes(char to_be_escaped, char escape, const string &val);
+	static bool RequiresQuotes(const char *str, idx_t len, const string &null_str, const vector<bool> &requires_quotes);
+	static void WriteQuotedString(WriteStream &writer, const char *str, idx_t len, bool force_quote,
+	                              const string &null_str, const vector<bool> &requires_quotes, char quote, char escape);
+	static void WriteQuotedString(WriteStream &writer, const char *str, idx_t len, idx_t col_idx,
+	                              CSVReaderOptions &options, CSVWriterOptions &writer_options);
+
+	static void WriteChunk(DataChunk &input, MemoryStream &writer, CSVReaderOptions &options, bool &written_anything,
+	                       CSVWriterOptions &writer_options);
+	static void WriteHeader(MemoryStream &stream, CSVReaderOptions &options, CSVWriterOptions &writer_options);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/encryption_functions.hpp

@@ -0,0 +1,54 @@
+#pragma once
+
+#include "duckdb/common/helper.hpp"
+
+namespace duckdb {
+
+class DatabaseInstance;
+class AttachedDatabase;
+class FileBuffer;
+
+struct EncryptionTag {
+	EncryptionTag();
+	data_ptr_t data();
+	idx_t size() const;
+
+private:
+	unique_ptr<data_t[]> tag;
+};
+
+struct EncryptionNonce {
+	EncryptionNonce();
+	data_ptr_t data();
+	idx_t size() const;
+
+private:
+	unique_ptr<data_t[]> nonce;
+};
+
+class EncryptionEngine {
+
+public:
+	EncryptionEngine();
+	~EncryptionEngine();
+
+public:
+	//! General key management wrapper functions
+	static const_data_ptr_t GetKeyFromCache(DatabaseInstance &db, const string &key_name);
+	static bool ContainsKey(DatabaseInstance &db, const string &key_name);
+	static void AddKeyToCache(DatabaseInstance &db, data_ptr_t key, const string &key_name, bool wipe = true);
+	static string AddKeyToCache(DatabaseInstance &db, data_ptr_t key);
+	static void AddTempKeyToCache(DatabaseInstance &db);
+
+	//! Encryption Functions
+	static void EncryptBlock(AttachedDatabase &attached_db, const string &key_id, FileBuffer &block,
+	                         FileBuffer &temp_buffer_manager, uint64_t delta);
+	static void DecryptBlock(AttachedDatabase &attached_db, const string &key_id, data_ptr_t internal_buffer,
+	                         uint64_t block_size, uint64_t delta);
+
+	static void EncryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t buffer, idx_t buffer_size, data_ptr_t metadata);
+
+	static void DecryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t buffer, idx_t buffer_size, data_ptr_t metadata);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/encryption_key_manager.hpp

@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/encryption_key_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/helper.hpp"
+#include "duckdb/common/types/value.hpp"
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/types.hpp"
+#include "duckdb/storage/object_cache.hpp"
+
+namespace duckdb {
+
+class EncryptionKey {
+
+public:
+	explicit EncryptionKey(data_ptr_t encryption_key);
+	~EncryptionKey();
+
+	EncryptionKey(const EncryptionKey &) = delete;
+	EncryptionKey &operator=(const EncryptionKey &) = delete;
+
+	EncryptionKey(EncryptionKey &&) noexcept = default;
+	EncryptionKey &operator=(EncryptionKey &&) noexcept = default;
+
+public:
+	const_data_ptr_t GetPtr() const {
+		return key;
+	}
+
+public:
+	static void LockEncryptionKey(data_ptr_t key, idx_t key_len = MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
+	static void UnlockEncryptionKey(data_ptr_t key, idx_t key_len = MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
+
+private:
+	data_t key[MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH];
+};
+
+class EncryptionKeyManager : public ObjectCacheEntry {
+
+public:
+	static EncryptionKeyManager &GetInternal(ObjectCache &cache);
+	static EncryptionKeyManager &Get(ClientContext &context);
+	static EncryptionKeyManager &Get(DatabaseInstance &db);
+
+public:
+	void AddKey(const string &key_name, data_ptr_t key);
+	bool HasKey(const string &key_name) const;
+	void DeleteKey(const string &key_name);
+	const_data_ptr_t GetKey(const string &key_name) const;
+
+public:
+	static string ObjectType();
+	string GetObjectType() override;
+
+public:
+public:
+	static void DeriveKey(string &user_key, data_ptr_t salt, data_ptr_t derived_key);
+	static void KeyDerivationFunctionSHA256(const_data_ptr_t user_key, idx_t user_key_size, data_ptr_t salt,
+	                                        data_ptr_t derived_key);
+	static void KeyDerivationFunctionSHA256(data_ptr_t user_key, idx_t user_key_size, data_ptr_t salt,
+	                                        data_ptr_t derived_key);
+	static string Base64Decode(const string &key);
+	static string GenerateRandomKeyID();
+
+public:
+	//! constants
+	static constexpr idx_t KEY_ID_BYTES = 8;
+	static constexpr idx_t DERIVED_KEY_LENGTH = 32;
+
+private:
+	std::unordered_map<std::string, EncryptionKey> derived_keys;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/encryption_state.hpp

@@ -9,26 +9,41 @@
 #pragma once
 
 #include "duckdb/common/helper.hpp"
-#include "duckdb/common/types/value.hpp"
+#include "duckdb/common/string_util.hpp"
 
 namespace duckdb {
 
+class EncryptionTypes {
+
+public:
+	enum CipherType : uint8_t { INVALID = 0, GCM = 1, CTR = 2, CBC = 3 };
+	enum KeyDerivationFunction : uint8_t { DEFAULT = 0, SHA256 = 1, PBKDF2 = 2 };
+	enum Mode { ENCRYPT, DECRYPT };
+
+	static string CipherToString(CipherType cipher_p);
+	static CipherType StringToCipher(const string &encryption_cipher_p);
+	static string KDFToString(KeyDerivationFunction kdf_p);
+	static KeyDerivationFunction StringToKDF(const string &key_derivation_function_p);
+};
+
 class EncryptionState {
 
 public:
-	DUCKDB_API explicit EncryptionState(const std::string *key = nullptr);
+	DUCKDB_API explicit EncryptionState(EncryptionTypes::CipherType cipher_p, idx_t key_len);
 	DUCKDB_API virtual ~EncryptionState();
 
 public:
-	DUCKDB_API virtual void InitializeEncryption(const_data_ptr_t iv, idx_t iv_len, const std::string *key);
-	DUCKDB_API virtual void InitializeDecryption(const_data_ptr_t iv, idx_t iv_len, const std::string *key);
+	DUCKDB_API virtual void InitializeEncryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len,
+	                                             const_data_ptr_t aad = nullptr, idx_t aad_len = 0);
+	DUCKDB_API virtual void InitializeDecryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len,
+	                                             const_data_ptr_t aad = nullptr, idx_t aad_len = 0);
 	DUCKDB_API virtual size_t Process(const_data_ptr_t in, idx_t in_len, data_ptr_t out, idx_t out_len);
 	DUCKDB_API virtual size_t Finalize(data_ptr_t out, idx_t out_len, data_ptr_t tag, idx_t tag_len);
 	DUCKDB_API virtual void GenerateRandomData(data_ptr_t data, idx_t len);
 
-public:
-	enum Mode { ENCRYPT, DECRYPT };
-	enum Cipher { GCM, CTR };
+protected:
+	EncryptionTypes::CipherType cipher;
+	idx_t key_len;
 };
 
 class EncryptionUtil {
@@ -37,8 +52,9 @@ public:
 	DUCKDB_API explicit EncryptionUtil() {};
 
 public:
-	virtual shared_ptr<EncryptionState> CreateEncryptionState(const std::string *key = nullptr) const {
-		return make_shared_ptr<EncryptionState>();
+	virtual shared_ptr<EncryptionState> CreateEncryptionState(EncryptionTypes::CipherType cipher_p,
+	                                                          idx_t key_len = 0) const {
+		return make_shared_ptr<EncryptionState>(cipher_p, key_len);
 	}
 
 	virtual ~EncryptionUtil() {