duckdb 0.10.2-dev0.0 → 0.10.2-dev5.0

package/src/duckdb/src/common/adbc/adbc.cpp

@@ -67,6 +67,8 @@ struct DuckDBAdbcStatementWrapper {
 	char *ingestion_table_name;
 	ArrowArrayStream ingestion_stream;
 	IngestionMode ingestion_mode = IngestionMode::CREATE;
+	uint8_t *substrait_plan;
+	uint64_t plan_length;
 };
 
 static AdbcStatusCode QueryInternal(struct AdbcConnection *connection, struct ArrowArrayStream *out, const char *query,
@@ -168,11 +170,10 @@ AdbcStatusCode StatementSetSubstraitPlan(struct AdbcStatement *statement, const
 		return ADBC_STATUS_INVALID_ARGUMENT;
 	}
 	auto wrapper = reinterpret_cast<DuckDBAdbcStatementWrapper *>(statement->private_data);
-	auto plan_str = std::string(reinterpret_cast<const char *>(plan), length);
-	auto query = "CALL from_substrait('" + plan_str + "'::BLOB)";
-	auto res = duckdb_prepare(wrapper->connection, query.c_str(), &wrapper->statement);
-	auto error_msg = duckdb_prepare_error(wrapper->statement);
-	return CheckResult(res, error, error_msg);
+	wrapper->substrait_plan = (uint8_t *)malloc(sizeof(uint8_t) * (length));
+	wrapper->plan_length = length;
+	memcpy(wrapper->substrait_plan, plan, length);
+	return ADBC_STATUS_OK;
 }
 
 AdbcStatusCode DatabaseSetOption(struct AdbcDatabase *database, const char *key, const char *value,
@@ -234,16 +235,11 @@ AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *connection, const
 		SetError(error, "Connection is not set");
 		return ADBC_STATUS_INVALID_ARGUMENT;
 	}
-	if (db_schema == nullptr) {
+	if (db_schema == nullptr || strlen(db_schema) == 0) {
 		// if schema is not set, we use the default schema
 		db_schema = "main";
 	}
-	if (catalog != nullptr && strlen(catalog) > 0) {
-		// In DuckDB this is the name of the database, not sure what's the expected functionality here, so for now,
-		// scream.
-		SetError(error, "Catalog Name is not used in DuckDB. It must be set to nullptr or an empty string");
-		return ADBC_STATUS_NOT_IMPLEMENTED;
-	} else if (table_name == nullptr) {
+	if (table_name == nullptr) {
 		SetError(error, "AdbcConnectionGetTableSchema: must provide table_name");
 		return ADBC_STATUS_INVALID_ARGUMENT;
 	} else if (strlen(table_name) == 0) {
@@ -253,9 +249,10 @@ AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *connection, const
 	ArrowArrayStream arrow_stream;
 
 	std::string query = "SELECT * FROM ";
-	if (strlen(db_schema) > 0) {
-		query += std::string(db_schema) + ".";
+	if (catalog != nullptr && strlen(catalog) > 0) {
+		query += std::string(catalog) + ".";
 	}
+	query += std::string(db_schema) + ".";
 	query += std::string(table_name) + " LIMIT 0;";
 
 	auto success = QueryInternal(connection, &arrow_stream, query.c_str(), error);
@@ -643,6 +640,8 @@ AdbcStatusCode StatementNew(struct AdbcConnection *connection, struct AdbcStatem
 	statement_wrapper->result = nullptr;
 	statement_wrapper->ingestion_stream.release = nullptr;
 	statement_wrapper->ingestion_table_name = nullptr;
+	statement_wrapper->substrait_plan = nullptr;
+
 	statement_wrapper->ingestion_mode = IngestionMode::CREATE;
 	return ADBC_STATUS_OK;
 }
@@ -668,6 +667,10 @@ AdbcStatusCode StatementRelease(struct AdbcStatement *statement, struct AdbcErro
 		free(wrapper->ingestion_table_name);
 		wrapper->ingestion_table_name = nullptr;
 	}
+	if (wrapper->substrait_plan) {
+		free(wrapper->substrait_plan);
+		wrapper->substrait_plan = nullptr;
+	}
 	free(statement->private_data);
 	statement->private_data = nullptr;
 	return ADBC_STATUS_OK;
@@ -758,8 +761,24 @@ AdbcStatusCode StatementExecuteQuery(struct AdbcStatement *statement, struct Arr
 	if (has_stream && to_table) {
 		return IngestToTableFromBoundStream(wrapper, error);
 	}
-
-	if (has_stream) {
+	if (wrapper->substrait_plan != nullptr) {
+		auto plan_str = std::string(reinterpret_cast<const char *>(wrapper->substrait_plan), wrapper->plan_length);
+		duckdb::vector<duckdb::Value> params;
+		params.emplace_back(duckdb::Value::BLOB_RAW(plan_str));
+		duckdb::unique_ptr<duckdb::QueryResult> query_result;
+		try {
+			query_result =
+			    ((duckdb::Connection *)wrapper->connection)->TableFunction("from_substrait", params)->Execute();
+		} catch (duckdb::Exception &e) {
+			std::string error_msg = "It was not possible to execute substrait query. " + std::string(e.what());
+			SetError(error, error_msg);
+			return ADBC_STATUS_INVALID_ARGUMENT;
+		}
+		auto arrow_wrapper = new duckdb::ArrowResultWrapper();
+		arrow_wrapper->result =
+		    duckdb::unique_ptr_cast<duckdb::QueryResult, duckdb::MaterializedQueryResult>(std::move(query_result));
+		wrapper->result = reinterpret_cast<duckdb_arrow>(arrow_wrapper);
+	} else if (has_stream) {
 		// A stream was bound to the statement, use that to bind parameters
 		duckdb::unique_ptr<duckdb::QueryResult> result;
 		ArrowArrayStream stream = wrapper->ingestion_stream;
@@ -960,57 +979,280 @@ AdbcStatusCode StatementSetOption(struct AdbcStatement *statement, const char *k
 AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth, const char *catalog,
                                     const char *db_schema, const char *table_name, const char **table_type,
                                     const char *column_name, struct ArrowArrayStream *out, struct AdbcError *error) {
-	if (catalog != nullptr) {
-		if (strcmp(catalog, "duckdb") == 0) {
-			SetError(error, "catalog must be NULL or 'duckdb'");
-			return ADBC_STATUS_INVALID_ARGUMENT;
-		}
-	}
-
 	if (table_type != nullptr) {
 		SetError(error, "Table types parameter not yet supported");
 		return ADBC_STATUS_NOT_IMPLEMENTED;
 	}
+
+	std::string catalog_filter = catalog ? catalog : "%";
+	std::string db_schema_filter = db_schema ? db_schema : "%";
+	std::string table_name_filter = table_name ? table_name : "%";
+	std::string column_name_filter = column_name ? column_name : "%";
+
 	std::string query;
 	switch (depth) {
 	case ADBC_OBJECT_DEPTH_CATALOGS:
-		SetError(error, "ADBC_OBJECT_DEPTH_CATALOGS not yet supported");
-		return ADBC_STATUS_NOT_IMPLEMENTED;
+		// Return metadata on catalogs.
+		query = duckdb::StringUtil::Format(R"(
+				SELECT
+					catalog_name,
+					[]::STRUCT(
+						db_schema_name VARCHAR,
+						db_schema_tables STRUCT(
+							table_name VARCHAR,
+							table_type VARCHAR,
+							table_columns STRUCT(
+								column_name VARCHAR,
+								ordinal_position INTEGER,
+								remarks VARCHAR,
+								xdbc_data_type SMALLINT,
+								xdbc_type_name VARCHAR,
+								xdbc_column_size INTEGER,
+								xdbc_decimal_digits SMALLINT,
+								xdbc_num_prec_radix SMALLINT,
+								xdbc_nullable SMALLINT,
+								xdbc_column_def VARCHAR,
+								xdbc_sql_data_type SMALLINT,
+								xdbc_datetime_sub SMALLINT,
+								xdbc_char_octet_length INTEGER,
+								xdbc_is_nullable VARCHAR,
+								xdbc_scope_catalog VARCHAR,
+								xdbc_scope_schema VARCHAR,
+								xdbc_scope_table VARCHAR,
+								xdbc_is_autoincrement BOOLEAN,
+								xdbc_is_generatedcolumn BOOLEAN
+							)[],
+							table_constraints STRUCT(
+								constraint_name VARCHAR,
+								constraint_type VARCHAR,
+								constraint_column_names VARCHAR[],
+								constraint_column_usage STRUCT(fk_catalog VARCHAR, fk_db_schema VARCHAR, fk_table VARCHAR, fk_column_name VARCHAR)[]
+							)[]
+						)[]
+					)[] catalog_db_schemas
+				FROM
+					information_schema.schemata
+				WHERE catalog_name LIKE '%s'
+				GROUP BY catalog_name
+				)",
+		                                   catalog_filter);
+		break;
 	case ADBC_OBJECT_DEPTH_DB_SCHEMAS:
 		// Return metadata on catalogs and schemas.
 		query = duckdb::StringUtil::Format(R"(
-				SELECT table_schema db_schema_name
-				FROM information_schema.columns
-				WHERE table_schema LIKE '%s' AND table_name LIKE '%s' AND column_name LIKE '%s' ;
+				WITH db_schemas AS (
+					SELECT
+						catalog_name,
+						schema_name,
+					FROM information_schema.schemata
+					WHERE schema_name LIKE '%s'
+				)
+
+				SELECT
+					catalog_name,
+					LIST({
+						db_schema_name: schema_name,
+						db_schema_tables: []::STRUCT(
+							table_name VARCHAR,
+							table_type VARCHAR,
+							table_columns STRUCT(
+								column_name VARCHAR,
+								ordinal_position INTEGER,
+								remarks VARCHAR,
+								xdbc_data_type SMALLINT,
+								xdbc_type_name VARCHAR,
+								xdbc_column_size INTEGER,
+								xdbc_decimal_digits SMALLINT,
+								xdbc_num_prec_radix SMALLINT,
+								xdbc_nullable SMALLINT,
+								xdbc_column_def VARCHAR,
+								xdbc_sql_data_type SMALLINT,
+								xdbc_datetime_sub SMALLINT,
+								xdbc_char_octet_length INTEGER,
+								xdbc_is_nullable VARCHAR,
+								xdbc_scope_catalog VARCHAR,
+								xdbc_scope_schema VARCHAR,
+								xdbc_scope_table VARCHAR,
+								xdbc_is_autoincrement BOOLEAN,
+								xdbc_is_generatedcolumn BOOLEAN
+							)[],
+							table_constraints STRUCT(
+								constraint_name VARCHAR,
+								constraint_type VARCHAR,
+								constraint_column_names VARCHAR[],
+								constraint_column_usage STRUCT(fk_catalog VARCHAR, fk_db_schema VARCHAR, fk_table VARCHAR, fk_column_name VARCHAR)[]
+							)[]
+						)[],
+					}) FILTER (dbs.schema_name is not null) catalog_db_schemas
+				FROM
+					information_schema.schemata
+				LEFT JOIN db_schemas dbs
+				USING (catalog_name, schema_name)
+				WHERE catalog_name LIKE '%s'
+				GROUP BY catalog_name
 				)",
-		                                   db_schema ? db_schema : "%", table_name ? table_name : "%",
-		                                   column_name ? column_name : "%");
+		                                   db_schema_filter, catalog_filter);
 		break;
 	case ADBC_OBJECT_DEPTH_TABLES:
 		// Return metadata on catalogs, schemas, and tables.
 		query = duckdb::StringUtil::Format(R"(
-				SELECT table_schema db_schema_name, LIST(table_schema_list) db_schema_tables
-				FROM (
-					SELECT table_schema, { table_name : table_name} table_schema_list
-					FROM information_schema.columns
-					WHERE table_schema LIKE '%s' AND table_name LIKE '%s' AND column_name LIKE '%s'  GROUP BY table_schema, table_name
-					) GROUP BY table_schema;
+				WITH tables AS (
+					SELECT
+						table_catalog catalog_name,
+						table_schema schema_name,
+						LIST({
+							table_name: table_name,
+							table_type: table_type,
+							table_columns: []::STRUCT(
+								column_name VARCHAR,
+								ordinal_position INTEGER,
+								remarks VARCHAR,
+								xdbc_data_type SMALLINT,
+								xdbc_type_name VARCHAR,
+								xdbc_column_size INTEGER,
+								xdbc_decimal_digits SMALLINT,
+								xdbc_num_prec_radix SMALLINT,
+								xdbc_nullable SMALLINT,
+								xdbc_column_def VARCHAR,
+								xdbc_sql_data_type SMALLINT,
+								xdbc_datetime_sub SMALLINT,
+								xdbc_char_octet_length INTEGER,
+								xdbc_is_nullable VARCHAR,
+								xdbc_scope_catalog VARCHAR,
+								xdbc_scope_schema VARCHAR,
+								xdbc_scope_table VARCHAR,
+								xdbc_is_autoincrement BOOLEAN,
+								xdbc_is_generatedcolumn BOOLEAN
+							)[],
+							table_constraints: []::STRUCT(
+								constraint_name VARCHAR,
+								constraint_type VARCHAR,
+								constraint_column_names VARCHAR[],
+								constraint_column_usage STRUCT(fk_catalog VARCHAR, fk_db_schema VARCHAR, fk_table VARCHAR, fk_column_name VARCHAR)[]
+							)[],
+						}) db_schema_tables
+					FROM information_schema.tables
+					WHERE table_name LIKE '%s'
+					GROUP BY table_catalog, table_schema
+				),
+				db_schemas AS (
+					SELECT
+						catalog_name,
+						schema_name,
+						db_schema_tables,
+					FROM information_schema.schemata
+					LEFT JOIN tables
+					USING (catalog_name, schema_name)
+					WHERE schema_name LIKE '%s'
+				)
+
+				SELECT
+					catalog_name,
+					LIST({
+						db_schema_name: schema_name,
+						db_schema_tables: db_schema_tables,
+					}) FILTER (dbs.schema_name is not null) catalog_db_schemas
+				FROM
+					information_schema.schemata
+				LEFT JOIN db_schemas dbs
+				USING (catalog_name, schema_name)
+				WHERE catalog_name LIKE '%s'
+				GROUP BY catalog_name
 				)",
-		                                   db_schema ? db_schema : "%", table_name ? table_name : "%",
-		                                   column_name ? column_name : "%");
+		                                   table_name_filter, db_schema_filter, catalog_filter);
 		break;
 	case ADBC_OBJECT_DEPTH_COLUMNS:
 		// Return metadata on catalogs, schemas, tables, and columns.
 		query = duckdb::StringUtil::Format(R"(
-				SELECT table_schema db_schema_name, LIST(table_schema_list) db_schema_tables
-				FROM (
-					SELECT table_schema, { table_name : table_name, table_columns : LIST({column_name : column_name, ordinal_position : ordinal_position + 1, remarks : ''})} table_schema_list
+				WITH columns AS (
+					SELECT
+						table_catalog,
+						table_schema,
+						table_name,
+						LIST({
+							column_name: column_name,
+							ordinal_position: ordinal_position,
+							remarks : '',
+							xdbc_data_type: NULL::SMALLINT,
+							xdbc_type_name: NULL::VARCHAR,
+							xdbc_column_size: NULL::INTEGER,
+							xdbc_decimal_digits: NULL::SMALLINT,
+							xdbc_num_prec_radix: NULL::SMALLINT,
+							xdbc_nullable: NULL::SMALLINT,
+							xdbc_column_def: NULL::VARCHAR,
+							xdbc_sql_data_type: NULL::SMALLINT,
+							xdbc_datetime_sub: NULL::SMALLINT,
+							xdbc_char_octet_length: NULL::INTEGER,
+							xdbc_is_nullable: NULL::VARCHAR,
+							xdbc_scope_catalog: NULL::VARCHAR,
+							xdbc_scope_schema: NULL::VARCHAR,
+							xdbc_scope_table: NULL::VARCHAR,
+							xdbc_is_autoincrement: NULL::BOOLEAN,
+							xdbc_is_generatedcolumn: NULL::BOOLEAN,
+						}) table_columns
 					FROM information_schema.columns
-					WHERE table_schema LIKE '%s' AND table_name LIKE '%s' AND column_name LIKE '%s' GROUP BY table_schema, table_name
-					) GROUP BY table_schema;
+					WHERE column_name LIKE '%s'
+					GROUP BY table_catalog, table_schema, table_name
+				),
+				constraints AS (
+					SELECT
+						table_catalog,
+						table_schema,
+						table_name,
+						LIST(
+							{
+								constraint_name: constraint_name,
+								constraint_type: constraint_type,
+								constraint_column_names: []::VARCHAR[],
+								constraint_column_usage: []::STRUCT(fk_catalog VARCHAR, fk_db_schema VARCHAR, fk_table VARCHAR, fk_column_name VARCHAR)[],
+							}
+						) table_constraints
+					FROM information_schema.table_constraints
+					GROUP BY table_catalog, table_schema, table_name
+				),
+				tables AS (
+					SELECT
+						table_catalog catalog_name,
+						table_schema schema_name,
+						LIST({
+							table_name: table_name,
+							table_type: table_type,
+							table_columns: table_columns,
+							table_constraints: table_constraints,
+						}) db_schema_tables
+					FROM information_schema.tables
+					LEFT JOIN columns
+					USING (table_catalog, table_schema, table_name)
+					LEFT JOIN constraints
+					USING (table_catalog, table_schema, table_name)
+					WHERE table_name LIKE '%s'
+					GROUP BY table_catalog, table_schema
+				),
+				db_schemas AS (
+					SELECT
+						catalog_name,
+						schema_name,
+						db_schema_tables,
+					FROM information_schema.schemata
+					LEFT JOIN tables
+					USING (catalog_name, schema_name)
+					WHERE schema_name LIKE '%s'
+				)
+
+				SELECT
+					catalog_name,
+					LIST({
+						db_schema_name: schema_name,
+						db_schema_tables: db_schema_tables,
+					}) FILTER (dbs.schema_name is not null) catalog_db_schemas
+				FROM
+					information_schema.schemata
+				LEFT JOIN db_schemas dbs
+				USING (catalog_name, schema_name)
+				WHERE catalog_name LIKE '%s'
+				GROUP BY catalog_name
 				)",
-		                                   db_schema ? db_schema : "%", table_name ? table_name : "%",
-		                                   column_name ? column_name : "%");
+		                                   column_name_filter, table_name_filter, db_schema_filter, catalog_filter);
 		break;
 	default:
 		SetError(error, "Invalid value of Depth");

package/src/duckdb/src/common/arrow/appender/union_data.cpp

@@ -55,8 +55,8 @@ void ArrowUnionData::Append(ArrowAppendData &append_data, Vector &input, idx_t f
 }
 
 void ArrowUnionData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
-	result->n_buffers = 2;
-	result->buffers[1] = append_data.main_buffer.data();
+	result->n_buffers = 1;
+	result->buffers[0] = append_data.main_buffer.data();
 
 	auto &child_types = UnionType::CopyMemberTypes(type);
 	ArrowAppender::AddChildren(append_data, child_types.size());

package/src/duckdb/src/common/box_renderer.cpp

@@ -446,7 +446,7 @@ void BoxRenderer::RenderHeader(const vector<string> &names, const vector<Logical
 		}
 	}
 	ss << config.RTCORNER;
-	ss << std::endl;
+	ss << '\n';
 
 	// render the header names
 	for (idx_t c = 0; c < column_count; c++) {
@@ -460,7 +460,7 @@ void BoxRenderer::RenderHeader(const vector<string> &names, const vector<Logical
 		RenderValue(ss, name, widths[c]);
 	}
 	ss << config.VERTICAL;
-	ss << std::endl;
+	ss << '\n';
 
 	// render the types
 	if (config.render_mode == RenderMode::ROWS) {
@@ -470,7 +470,7 @@ void BoxRenderer::RenderHeader(const vector<string> &names, const vector<Logical
 			RenderValue(ss, type, widths[c]);
 		}
 		ss << config.VERTICAL;
-		ss << std::endl;
+		ss << '\n';
 	}
 
 	// render the line under the header
@@ -485,7 +485,7 @@ void BoxRenderer::RenderHeader(const vector<string> &names, const vector<Logical
 		}
 	}
 	ss << config.RMIDDLE;
-	ss << std::endl;
+	ss << '\n';
 }
 
 void BoxRenderer::RenderValues(const list<ColumnDataCollection> &collections, const vector<idx_t> &column_map,
@@ -534,7 +534,7 @@ void BoxRenderer::RenderValues(const list<ColumnDataCollection> &collections, co
 			RenderValue(ss, str, widths[c], alignment);
 		}
 		ss << config.VERTICAL;
-		ss << std::endl;
+		ss << '\n';
 	}
 
 	if (bottom_rows > 0) {
@@ -590,7 +590,7 @@ void BoxRenderer::RenderValues(const list<ColumnDataCollection> &collections, co
 				RenderValue(ss, str, widths[c], alignment);
 			}
 			ss << config.VERTICAL;
-			ss << std::endl;
+			ss << '\n';
 		}
 		// note that the bottom rows are in reverse order
 		for (idx_t r = 0; r < bottom_rows; r++) {
@@ -605,7 +605,7 @@ void BoxRenderer::RenderValues(const list<ColumnDataCollection> &collections, co
 				RenderValue(ss, str, widths[c], alignments[c]);
 			}
 			ss << config.VERTICAL;
-			ss << std::endl;
+			ss << '\n';
 		}
 	}
 }
@@ -644,7 +644,7 @@ void BoxRenderer::RenderRowCount(string row_count_str, string shown_str, const s
 			}
 		}
 		ss << (render_anything ? config.RMIDDLE : config.RDCORNER);
-		ss << std::endl;
+		ss << '\n';
 	}
 	if (!render_anything) {
 		return;
@@ -658,16 +658,16 @@ void BoxRenderer::RenderRowCount(string row_count_str, string shown_str, const s
 		ss << column_count_str;
 		ss << " ";
 		ss << config.VERTICAL;
-		ss << std::endl;
+		ss << '\n';
 	} else if (render_rows) {
 		RenderValue(ss, row_count_str, total_length - 4);
 		ss << config.VERTICAL;
-		ss << std::endl;
+		ss << '\n';
 
 		if (display_shown_separately) {
 			RenderValue(ss, shown_str, total_length - 4);
 			ss << config.VERTICAL;
-			ss << std::endl;
+			ss << '\n';
 		}
 	}
 	// render the bottom line
@@ -676,7 +676,7 @@ void BoxRenderer::RenderRowCount(string row_count_str, string shown_str, const s
 		ss << config.HORIZONTAL;
 	}
 	ss << config.RDCORNER;
-	ss << std::endl;
+	ss << '\n';
 }
 
 void BoxRenderer::Render(ClientContext &context, const vector<string> &names, const ColumnDataCollection &result,

package/src/duckdb/src/common/crypto/md5.cpp

@@ -156,7 +156,8 @@ void MD5Context::MD5Update(const_data_ptr_t input, idx_t len) {
 	/* Update bitcount */
 
 	t = bits[0];
-	if ((bits[0] = t + ((uint32_t)len << 3)) < t) {
+	bits[0] = t + ((uint32_t)len << 3);
+	if (bits[0] < t) {
 		bits[1]++; /* Carry from low to high */
 	}
 	bits[1] += len >> 29;