duckdb 0.3.2-dev1288.0 → 0.3.2-dev1306.0

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "0d3fee81b"
-#define DUCKDB_VERSION "v0.3.2-dev1288"
+#define DUCKDB_SOURCE_ID "8835c86b4"
+#define DUCKDB_VERSION "v0.3.2-dev1306"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -3323,6 +3323,21 @@ enum class VectorBufferType : uint8_t {
 	OPAQUE_BUFFER        // opaque buffer, can be created for example by the parquet reader
 };
 
+enum class VectorAuxiliaryDataType : uint8_t {
+	ARROW_AUXILIARY // Holds Arrow Chunks that this vector depends on
+};
+
+struct VectorAuxiliaryData {
+	explicit VectorAuxiliaryData(VectorAuxiliaryDataType type_p)
+	    : type(type_p) {
+
+	      };
+	VectorAuxiliaryDataType type;
+
+	virtual ~VectorAuxiliaryData() {
+	}
+};
+
 //! The VectorBuffer is a class used by the vector to hold its data
 class VectorBuffer {
 public:
@@ -3345,10 +3360,19 @@ public:
 	data_ptr_t GetData() {
 		return data.get();
 	}
+
 	void SetData(unique_ptr<data_t[]> new_data) {
 		data = move(new_data);
 	}
 
+	VectorAuxiliaryData *GetAuxiliaryData() {
+		return aux_data.get();
+	}
+
+	void SetAuxiliaryData(unique_ptr<VectorAuxiliaryData> aux_data_p, VectorAuxiliaryDataType aux_type_p) {
+		aux_data = move(aux_data_p);
+	}
+
 	static buffer_ptr<VectorBuffer> CreateStandardVector(PhysicalType type, idx_t capacity = STANDARD_VECTOR_SIZE);
 	static buffer_ptr<VectorBuffer> CreateConstantVector(PhysicalType type);
 	static buffer_ptr<VectorBuffer> CreateConstantVector(const LogicalType &logical_type);
@@ -3359,8 +3383,13 @@ public:
 		return buffer_type;
 	}
 
+	inline VectorAuxiliaryDataType GetAuxiliaryDataType() const {
+		return aux_data->type;
+	}
+
 protected:
 	VectorBufferType buffer_type;
+	unique_ptr<VectorAuxiliaryData> aux_data;
 	unique_ptr<data_t[]> data;
 };
 
@@ -3846,6 +3875,141 @@ struct SequenceVector {
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/arrow_wrapper.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/arrow.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ARROW_FLAG_DICTIONARY_ORDERED
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define ARROW_FLAG_DICTIONARY_ORDERED 1
+#define ARROW_FLAG_NULLABLE           2
+#define ARROW_FLAG_MAP_KEYS_SORTED    4
+
+struct ArrowSchema {
+	// Array type description
+	const char *format;
+	const char *name;
+	const char *metadata;
+	int64_t flags;
+	int64_t n_children;
+	struct ArrowSchema **children;
+	struct ArrowSchema *dictionary;
+
+	// Release callback
+	void (*release)(struct ArrowSchema *);
+	// Opaque producer-specific data
+	void *private_data;
+};
+
+struct ArrowArray {
+	// Array data description
+	int64_t length;
+	int64_t null_count;
+	int64_t offset;
+	int64_t n_buffers;
+	int64_t n_children;
+	const void **buffers;
+	struct ArrowArray **children;
+	struct ArrowArray *dictionary;
+
+	// Release callback
+	void (*release)(struct ArrowArray *);
+	// Opaque producer-specific data
+	void *private_data;
+};
+
+// EXPERIMENTAL
+struct ArrowArrayStream {
+	// Callback to get the stream type
+	// (will be the same for all arrays in the stream).
+	// Return value: 0 if successful, an `errno`-compatible error code otherwise.
+	int (*get_schema)(struct ArrowArrayStream *, struct ArrowSchema *out);
+	// Callback to get the next array
+	// (if no error and the array is released, the stream has ended)
+	// Return value: 0 if successful, an `errno`-compatible error code otherwise.
+	int (*get_next)(struct ArrowArrayStream *, struct ArrowArray *out);
+
+	// Callback to get optional detailed error information.
+	// This must only be called if the last stream operation failed
+	// with a non-0 return code.  The returned pointer is only valid until
+	// the next operation on this stream (including release).
+	// If unavailable, NULL is returned.
+	const char *(*get_last_error)(struct ArrowArrayStream *);
+
+	// Release callback: release the stream's own resources.
+	// Note that arrays returned by `get_next` must be individually released.
+	void (*release)(struct ArrowArrayStream *);
+	// Opaque producer-specific data
+	void *private_data;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
+
+
+//! Here we have the internal duckdb classes that interact with Arrow's Internal Header (i.e., duckdb/commons/arrow.hpp)
+namespace duckdb {
+class ArrowSchemaWrapper {
+public:
+	ArrowSchema arrow_schema;
+
+	ArrowSchemaWrapper() {
+		arrow_schema.release = nullptr;
+	}
+
+	~ArrowSchemaWrapper();
+};
+class ArrowArrayWrapper {
+public:
+	ArrowArray arrow_array;
+	ArrowArrayWrapper() {
+		arrow_array.length = 0;
+		arrow_array.release = nullptr;
+	}
+	~ArrowArrayWrapper();
+};
+
+class ArrowArrayStreamWrapper {
+public:
+	ArrowArrayStream arrow_array_stream;
+	int64_t number_of_rows;
+	void GetSchema(ArrowSchemaWrapper &schema);
+
+	shared_ptr<ArrowArrayWrapper> GetNextChunk();
+
+	const char *GetError();
+
+	~ArrowArrayStreamWrapper();
+	ArrowArrayStreamWrapper() {
+		arrow_array_stream.release = nullptr;
+	}
+};
+} // namespace duckdb
+
+
 struct ArrowArray;
 
 namespace duckdb {
@@ -12871,89 +13035,6 @@ private:
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/arrow.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef ARROW_FLAG_DICTIONARY_ORDERED
-
-#include <stdint.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define ARROW_FLAG_DICTIONARY_ORDERED 1
-#define ARROW_FLAG_NULLABLE           2
-#define ARROW_FLAG_MAP_KEYS_SORTED    4
-
-struct ArrowSchema {
-	// Array type description
-	const char *format;
-	const char *name;
-	const char *metadata;
-	int64_t flags;
-	int64_t n_children;
-	struct ArrowSchema **children;
-	struct ArrowSchema *dictionary;
-
-	// Release callback
-	void (*release)(struct ArrowSchema *);
-	// Opaque producer-specific data
-	void *private_data;
-};
-
-struct ArrowArray {
-	// Array data description
-	int64_t length;
-	int64_t null_count;
-	int64_t offset;
-	int64_t n_buffers;
-	int64_t n_children;
-	const void **buffers;
-	struct ArrowArray **children;
-	struct ArrowArray *dictionary;
-
-	// Release callback
-	void (*release)(struct ArrowArray *);
-	// Opaque producer-specific data
-	void *private_data;
-};
-
-// EXPERIMENTAL
-struct ArrowArrayStream {
-	// Callback to get the stream type
-	// (will be the same for all arrays in the stream).
-	// Return value: 0 if successful, an `errno`-compatible error code otherwise.
-	int (*get_schema)(struct ArrowArrayStream *, struct ArrowSchema *out);
-	// Callback to get the next array
-	// (if no error and the array is released, the stream has ended)
-	// Return value: 0 if successful, an `errno`-compatible error code otherwise.
-	int (*get_next)(struct ArrowArrayStream *, struct ArrowArray *out);
-
-	// Callback to get optional detailed error information.
-	// This must only be called if the last stream operation failed
-	// with a non-0 return code.  The returned pointer is only valid until
-	// the next operation on this stream (including release).
-	// If unavailable, NULL is returned.
-	const char *(*get_last_error)(struct ArrowArrayStream *);
-
-	// Release callback: release the stream's own resources.
-	// Note that arrays returned by `get_next` must be individually released.
-	void (*release)(struct ArrowArrayStream *);
-	// Opaque producer-specific data
-	void *private_data;
-};
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
 // duckdb/common/types/blob.hpp
 //
 //
@@ -20654,6 +20735,9 @@ struct BufferedCSVReaderOptions {
 	idx_t buffer_size = STANDARD_VECTOR_SIZE * 100;
 	//! Consider all columns to be of type varchar
 	bool all_varchar = false;
+	//! Maximum CSV line size: specified because if we reach this amount, we likely have wrong delimiters (default: 2MB)
+	idx_t maximum_line_size = 2097152;
+
 	//! The date format to use (if any is specified)
 	std::map<LogicalTypeId, StrpTimeFormat> date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}};
 	//! Whether or not a type format is specified
@@ -20670,8 +20754,6 @@ enum class ParserMode : uint8_t { PARSING = 0, SNIFFING_DIALECT = 1, SNIFFING_DA
 class BufferedCSVReader {
 	//! Initial buffer read size; can be extended for long lines
 	static constexpr idx_t INITIAL_BUFFER_SIZE = 16384;
-	//! Maximum CSV line size: specified because if we reach this amount, we likely have the wrong delimiters
-	static constexpr idx_t MAXIMUM_CSV_LINE_SIZE = 1048576;
 	ParserMode mode;
 
 public:
@@ -20854,73 +20936,6 @@ public:
 	string ToString(const string &column_name) override;
 };
 
-} // namespace duckdb
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/common/arrow_wrapper.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-
-//! Here we have the internal duckdb classes that interact with Arrow's Internal Header (i.e., duckdb/commons/arrow.hpp)
-namespace duckdb {
-class ArrowSchemaWrapper {
-public:
-	ArrowSchema arrow_schema;
-
-	ArrowSchemaWrapper() {
-		arrow_schema.release = nullptr;
-	}
-
-	~ArrowSchemaWrapper();
-};
-class ArrowArrayWrapper {
-public:
-	ArrowArray arrow_array;
-	ArrowArrayWrapper() {
-		arrow_array.length = 0;
-		arrow_array.release = nullptr;
-	}
-	~ArrowArrayWrapper();
-};
-
-class ArrowArrayStreamWrapper {
-public:
-	ArrowArrayStream arrow_array_stream;
-	int64_t number_of_rows;
-	void GetSchema(ArrowSchemaWrapper &schema);
-
-	unique_ptr<ArrowArrayWrapper> GetNextChunk();
-
-	const char *GetError();
-
-	~ArrowArrayStreamWrapper();
-	ArrowArrayStreamWrapper() {
-		arrow_array_stream.release = nullptr;
-	}
-};
-
-class ResultArrowArrayStreamWrapper {
-public:
-	explicit ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result, idx_t approx_batch_size);
-	ArrowArrayStream stream;
-	unique_ptr<QueryResult> result;
-	std::string last_error;
-	idx_t vectors_per_chunk;
-
-private:
-	static int MyStreamGetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out);
-	static int MyStreamGetNext(struct ArrowArrayStream *stream, struct ArrowArray *out);
-	static void MyStreamRelease(struct ArrowArrayStream *stream);
-	static const char *MyStreamGetLastError(struct ArrowArrayStream *stream);
-};
-
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB