duckdb 0.3.2-dev780.0 → 0.3.2-dev816.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb.cpp +314 -15
- package/src/duckdb.hpp +30 -12
- package/src/parquet-amalgamation.cpp +29732 -29627
- package/src/parquet-amalgamation.hpp +3 -2
package/package.json
CHANGED
package/src/duckdb.cpp
CHANGED
|
@@ -38204,6 +38204,9 @@ LogicalType LogicalType::STRUCT(child_list_t<LogicalType> children) {
|
|
|
38204
38204
|
return LogicalType(LogicalTypeId::STRUCT, move(info));
|
|
38205
38205
|
}
|
|
38206
38206
|
|
|
38207
|
+
//===--------------------------------------------------------------------===//
|
|
38208
|
+
// Map Type
|
|
38209
|
+
//===--------------------------------------------------------------------===//
|
|
38207
38210
|
LogicalType LogicalType::MAP(child_list_t<LogicalType> children) {
|
|
38208
38211
|
auto info = make_shared<StructTypeInfo>(move(children));
|
|
38209
38212
|
return LogicalType(LogicalTypeId::MAP, move(info));
|
|
@@ -38216,6 +38219,16 @@ LogicalType LogicalType::MAP(LogicalType key, LogicalType value) {
|
|
|
38216
38219
|
return LogicalType::MAP(move(child_types));
|
|
38217
38220
|
}
|
|
38218
38221
|
|
|
38222
|
+
const LogicalType &MapType::KeyType(const LogicalType &type) {
|
|
38223
|
+
D_ASSERT(type.id() == LogicalTypeId::MAP);
|
|
38224
|
+
return StructType::GetChildTypes(type)[0].second;
|
|
38225
|
+
}
|
|
38226
|
+
|
|
38227
|
+
const LogicalType &MapType::ValueType(const LogicalType &type) {
|
|
38228
|
+
D_ASSERT(type.id() == LogicalTypeId::MAP);
|
|
38229
|
+
return StructType::GetChildTypes(type)[1].second;
|
|
38230
|
+
}
|
|
38231
|
+
|
|
38219
38232
|
//===--------------------------------------------------------------------===//
|
|
38220
38233
|
// User Type
|
|
38221
38234
|
//===--------------------------------------------------------------------===//
|
|
@@ -74467,7 +74480,8 @@ string MacroFunction::ValidateArguments(MacroCatalogEntry ¯o_func, FunctionE
|
|
|
74467
74480
|
} else if (defaults.find(arg->alias) != defaults.end()) {
|
|
74468
74481
|
return StringUtil::Format("Duplicate default parameters %s!", arg->alias);
|
|
74469
74482
|
}
|
|
74470
|
-
|
|
74483
|
+
auto alias = arg->alias;
|
|
74484
|
+
defaults[alias] = move(arg);
|
|
74471
74485
|
} else if (!defaults.empty()) {
|
|
74472
74486
|
return "Positional parameters cannot come after parameters with a default value!";
|
|
74473
74487
|
} else {
|
|
@@ -99884,6 +99898,7 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
|
|
|
99884
99898
|
|
|
99885
99899
|
|
|
99886
99900
|
|
|
99901
|
+
#ifndef DISABLE_DUCKDB_REMOTE_INSTALL
|
|
99887
99902
|
|
|
99888
99903
|
|
|
99889
99904
|
// LICENSE_CHANGE_BEGIN
|
|
@@ -107326,6 +107341,7 @@ inline SSL_CTX *Client::ssl_context() const {
|
|
|
107326
107341
|
|
|
107327
107342
|
// LICENSE_CHANGE_END
|
|
107328
107343
|
|
|
107344
|
+
#endif
|
|
107329
107345
|
|
|
107330
107346
|
|
|
107331
107347
|
#include <fstream>
|
|
@@ -107383,6 +107399,9 @@ void ExtensionHelper::InstallExtension(DatabaseInstance &db, const string &exten
|
|
|
107383
107399
|
throw IOException("Failed to read extension from \"%s\": no such file", extension);
|
|
107384
107400
|
}
|
|
107385
107401
|
|
|
107402
|
+
#ifdef DISABLE_DUCKDB_REMOTE_INSTALL
|
|
107403
|
+
throw BinderException("Remote extension installation is disabled through configuration");
|
|
107404
|
+
#else
|
|
107386
107405
|
string url_template = "http://extensions.duckdb.org/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension.gz";
|
|
107387
107406
|
|
|
107388
107407
|
if (is_http_url) {
|
|
@@ -107421,6 +107440,7 @@ void ExtensionHelper::InstallExtension(DatabaseInstance &db, const string &exten
|
|
|
107421
107440
|
if (out.bad()) {
|
|
107422
107441
|
throw IOException("Failed to write extension to %s", local_extension_path);
|
|
107423
107442
|
}
|
|
107443
|
+
#endif
|
|
107424
107444
|
}
|
|
107425
107445
|
|
|
107426
107446
|
} // namespace duckdb
|
|
@@ -112829,7 +112849,10 @@ void ExpressionRewriter::VisitExpression(unique_ptr<Expression> *expression) {
|
|
|
112829
112849
|
|
|
112830
112850
|
|
|
112831
112851
|
|
|
112852
|
+
|
|
112853
|
+
|
|
112832
112854
|
#include <functional>
|
|
112855
|
+
#include <map>
|
|
112833
112856
|
|
|
112834
112857
|
namespace duckdb {
|
|
112835
112858
|
|
|
@@ -112867,6 +112890,37 @@ private:
|
|
|
112867
112890
|
idx_t GetEquivalenceSet(Expression *expr);
|
|
112868
112891
|
FilterResult AddConstantComparison(vector<ExpressionValueInformation> &info_list, ExpressionValueInformation info);
|
|
112869
112892
|
|
|
112893
|
+
//! Functions used to push and generate OR Filters
|
|
112894
|
+
void LookUpConjunctions(Expression *expr);
|
|
112895
|
+
bool BFSLookUpConjunctions(BoundConjunctionExpression *conjunction);
|
|
112896
|
+
void VerifyOrsToPush(Expression &expr);
|
|
112897
|
+
|
|
112898
|
+
bool UpdateConjunctionFilter(BoundComparisonExpression *comparison_expr);
|
|
112899
|
+
bool UpdateFilterByColumn(BoundColumnRefExpression *column_ref, BoundComparisonExpression *comparison_expr);
|
|
112900
|
+
void GenerateORFilters(TableFilterSet &table_filter, vector<idx_t> &column_ids);
|
|
112901
|
+
|
|
112902
|
+
template <typename CONJUNCTION_TYPE>
|
|
112903
|
+
void GenerateConjunctionFilter(BoundConjunctionExpression *conjunction, ConjunctionFilter *last_conj_filter) {
|
|
112904
|
+
auto new_filter = NextConjunctionFilter<CONJUNCTION_TYPE>(conjunction);
|
|
112905
|
+
auto conj_filter_ptr = (ConjunctionFilter *)new_filter.get();
|
|
112906
|
+
last_conj_filter->child_filters.push_back(move(new_filter));
|
|
112907
|
+
last_conj_filter = conj_filter_ptr;
|
|
112908
|
+
}
|
|
112909
|
+
|
|
112910
|
+
template <typename CONJUNCTION_TYPE>
|
|
112911
|
+
unique_ptr<TableFilter> NextConjunctionFilter(BoundConjunctionExpression *conjunction) {
|
|
112912
|
+
unique_ptr<ConjunctionFilter> conj_filter = make_unique<CONJUNCTION_TYPE>();
|
|
112913
|
+
for (auto &expr : conjunction->children) {
|
|
112914
|
+
auto comp_expr = (BoundComparisonExpression *)expr.get();
|
|
112915
|
+
auto &const_expr =
|
|
112916
|
+
(comp_expr->left->type == ExpressionType::VALUE_CONSTANT) ? *comp_expr->left : *comp_expr->right;
|
|
112917
|
+
auto const_value = ExpressionExecutor::EvaluateScalar(const_expr);
|
|
112918
|
+
auto const_filter = make_unique<ConstantFilter>(comp_expr->type, const_value);
|
|
112919
|
+
conj_filter->child_filters.push_back(move(const_filter));
|
|
112920
|
+
}
|
|
112921
|
+
return move(conj_filter);
|
|
112922
|
+
}
|
|
112923
|
+
|
|
112870
112924
|
private:
|
|
112871
112925
|
vector<unique_ptr<Expression>> remaining_filters;
|
|
112872
112926
|
|
|
@@ -112875,6 +112929,26 @@ private:
|
|
|
112875
112929
|
unordered_map<idx_t, vector<ExpressionValueInformation>> constant_values;
|
|
112876
112930
|
unordered_map<idx_t, vector<Expression *>> equivalence_map;
|
|
112877
112931
|
idx_t set_index = 0;
|
|
112932
|
+
|
|
112933
|
+
//! Structures used for OR Filters
|
|
112934
|
+
|
|
112935
|
+
struct ConjunctionsToPush {
|
|
112936
|
+
BoundConjunctionExpression *root_or;
|
|
112937
|
+
|
|
112938
|
+
// only preserve AND if there is a single column in the expression
|
|
112939
|
+
bool preserve_and = true;
|
|
112940
|
+
|
|
112941
|
+
// conjunction chain for this column
|
|
112942
|
+
vector<unique_ptr<BoundConjunctionExpression>> conjunctions;
|
|
112943
|
+
};
|
|
112944
|
+
|
|
112945
|
+
expression_map_t<vector<unique_ptr<ConjunctionsToPush>>> map_col_conjunctions;
|
|
112946
|
+
vector<BoundColumnRefExpression *> vec_colref_insertion_order;
|
|
112947
|
+
|
|
112948
|
+
BoundConjunctionExpression *cur_root_or;
|
|
112949
|
+
BoundConjunctionExpression *cur_conjunction;
|
|
112950
|
+
|
|
112951
|
+
BoundColumnRefExpression *cur_colref_to_push;
|
|
112878
112952
|
};
|
|
112879
112953
|
|
|
112880
112954
|
} // namespace duckdb
|
|
@@ -112896,6 +112970,7 @@ private:
|
|
|
112896
112970
|
|
|
112897
112971
|
|
|
112898
112972
|
|
|
112973
|
+
|
|
112899
112974
|
namespace duckdb {
|
|
112900
112975
|
|
|
112901
112976
|
using ExpressionValueInformation = FilterCombiner::ExpressionValueInformation;
|
|
@@ -112958,6 +113033,7 @@ FilterResult FilterCombiner::AddConstantComparison(vector<ExpressionValueInforma
|
|
|
112958
113033
|
}
|
|
112959
113034
|
|
|
112960
113035
|
FilterResult FilterCombiner::AddFilter(unique_ptr<Expression> expr) {
|
|
113036
|
+
LookUpConjunctions(expr.get());
|
|
112961
113037
|
// try to push the filter into the combiner
|
|
112962
113038
|
auto result = AddFilter(expr.get());
|
|
112963
113039
|
if (result == FilterResult::UNSUPPORTED) {
|
|
@@ -113430,6 +113506,8 @@ TableFilterSet FilterCombiner::GenerateTableScanFilters(vector<idx_t> &column_id
|
|
|
113430
113506
|
}
|
|
113431
113507
|
}
|
|
113432
113508
|
|
|
113509
|
+
GenerateORFilters(table_filters, column_ids);
|
|
113510
|
+
|
|
113433
113511
|
return table_filters;
|
|
113434
113512
|
}
|
|
113435
113513
|
|
|
@@ -113867,6 +113945,176 @@ ValueComparisonResult CompareValueInformation(ExpressionValueInformation &left,
|
|
|
113867
113945
|
}
|
|
113868
113946
|
}
|
|
113869
113947
|
|
|
113948
|
+
void FilterCombiner::LookUpConjunctions(Expression *expr) {
|
|
113949
|
+
if (expr->GetExpressionType() == ExpressionType::CONJUNCTION_OR) {
|
|
113950
|
+
auto root_or_expr = (BoundConjunctionExpression *)expr;
|
|
113951
|
+
for (const auto &entry : map_col_conjunctions) {
|
|
113952
|
+
for (const auto &conjs_to_push : entry.second) {
|
|
113953
|
+
if (conjs_to_push->root_or->Equals(root_or_expr)) {
|
|
113954
|
+
return;
|
|
113955
|
+
}
|
|
113956
|
+
}
|
|
113957
|
+
}
|
|
113958
|
+
|
|
113959
|
+
cur_root_or = root_or_expr;
|
|
113960
|
+
cur_conjunction = root_or_expr;
|
|
113961
|
+
cur_colref_to_push = nullptr;
|
|
113962
|
+
if (!BFSLookUpConjunctions(cur_root_or)) {
|
|
113963
|
+
if (cur_colref_to_push) {
|
|
113964
|
+
auto entry = map_col_conjunctions.find(cur_colref_to_push);
|
|
113965
|
+
auto &vec_conjs_to_push = entry->second;
|
|
113966
|
+
if (vec_conjs_to_push.size() == 1) {
|
|
113967
|
+
map_col_conjunctions.erase(entry);
|
|
113968
|
+
return;
|
|
113969
|
+
}
|
|
113970
|
+
vec_conjs_to_push.pop_back();
|
|
113971
|
+
}
|
|
113972
|
+
}
|
|
113973
|
+
return;
|
|
113974
|
+
}
|
|
113975
|
+
|
|
113976
|
+
// Verify if the expression has a column already pushed down by other OR expression
|
|
113977
|
+
VerifyOrsToPush(*expr);
|
|
113978
|
+
}
|
|
113979
|
+
|
|
113980
|
+
bool FilterCombiner::BFSLookUpConjunctions(BoundConjunctionExpression *conjunction) {
|
|
113981
|
+
vector<BoundConjunctionExpression *> conjunctions_to_visit;
|
|
113982
|
+
|
|
113983
|
+
for (auto &child : conjunction->children) {
|
|
113984
|
+
switch (child->GetExpressionClass()) {
|
|
113985
|
+
case ExpressionClass::BOUND_CONJUNCTION: {
|
|
113986
|
+
auto child_conjunction = (BoundConjunctionExpression *)child.get();
|
|
113987
|
+
conjunctions_to_visit.emplace_back(child_conjunction);
|
|
113988
|
+
break;
|
|
113989
|
+
}
|
|
113990
|
+
case ExpressionClass::BOUND_COMPARISON: {
|
|
113991
|
+
if (!UpdateConjunctionFilter((BoundComparisonExpression *)child.get())) {
|
|
113992
|
+
return false;
|
|
113993
|
+
}
|
|
113994
|
+
break;
|
|
113995
|
+
}
|
|
113996
|
+
default: {
|
|
113997
|
+
return false;
|
|
113998
|
+
}
|
|
113999
|
+
}
|
|
114000
|
+
}
|
|
114001
|
+
|
|
114002
|
+
for (auto child_conjunction : conjunctions_to_visit) {
|
|
114003
|
+
cur_conjunction = child_conjunction;
|
|
114004
|
+
// traverse child conjuction
|
|
114005
|
+
if (!BFSLookUpConjunctions(child_conjunction)) {
|
|
114006
|
+
return false;
|
|
114007
|
+
}
|
|
114008
|
+
}
|
|
114009
|
+
return true;
|
|
114010
|
+
}
|
|
114011
|
+
|
|
114012
|
+
void FilterCombiner::VerifyOrsToPush(Expression &expr) {
|
|
114013
|
+
if (expr.type == ExpressionType::BOUND_COLUMN_REF) {
|
|
114014
|
+
auto colref = (BoundColumnRefExpression *)&expr;
|
|
114015
|
+
auto entry = map_col_conjunctions.find(colref);
|
|
114016
|
+
if (entry == map_col_conjunctions.end()) {
|
|
114017
|
+
return;
|
|
114018
|
+
}
|
|
114019
|
+
map_col_conjunctions.erase(entry);
|
|
114020
|
+
}
|
|
114021
|
+
ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { VerifyOrsToPush(child); });
|
|
114022
|
+
}
|
|
114023
|
+
|
|
114024
|
+
bool FilterCombiner::UpdateConjunctionFilter(BoundComparisonExpression *comparison_expr) {
|
|
114025
|
+
bool left_is_scalar = comparison_expr->left->IsFoldable();
|
|
114026
|
+
bool right_is_scalar = comparison_expr->right->IsFoldable();
|
|
114027
|
+
|
|
114028
|
+
Expression *non_scalar_expr;
|
|
114029
|
+
if (left_is_scalar || right_is_scalar) {
|
|
114030
|
+
// only support comparison with scalar
|
|
114031
|
+
non_scalar_expr = left_is_scalar ? comparison_expr->right.get() : comparison_expr->left.get();
|
|
114032
|
+
|
|
114033
|
+
if (non_scalar_expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) {
|
|
114034
|
+
return UpdateFilterByColumn((BoundColumnRefExpression *)non_scalar_expr, comparison_expr);
|
|
114035
|
+
}
|
|
114036
|
+
}
|
|
114037
|
+
|
|
114038
|
+
return false;
|
|
114039
|
+
}
|
|
114040
|
+
|
|
114041
|
+
bool FilterCombiner::UpdateFilterByColumn(BoundColumnRefExpression *column_ref,
|
|
114042
|
+
BoundComparisonExpression *comparison_expr) {
|
|
114043
|
+
if (cur_colref_to_push == nullptr) {
|
|
114044
|
+
cur_colref_to_push = column_ref;
|
|
114045
|
+
|
|
114046
|
+
auto or_conjunction = make_unique<BoundConjunctionExpression>(ExpressionType::CONJUNCTION_OR);
|
|
114047
|
+
or_conjunction->children.emplace_back(comparison_expr->Copy());
|
|
114048
|
+
|
|
114049
|
+
unique_ptr<ConjunctionsToPush> conjs_to_push = make_unique<ConjunctionsToPush>();
|
|
114050
|
+
conjs_to_push->conjunctions.emplace_back(move(or_conjunction));
|
|
114051
|
+
conjs_to_push->root_or = cur_root_or;
|
|
114052
|
+
|
|
114053
|
+
auto &&vec_col_conjs = map_col_conjunctions[column_ref];
|
|
114054
|
+
vec_col_conjs.emplace_back(move(conjs_to_push));
|
|
114055
|
+
vec_colref_insertion_order.emplace_back(column_ref);
|
|
114056
|
+
return true;
|
|
114057
|
+
}
|
|
114058
|
+
|
|
114059
|
+
auto entry = map_col_conjunctions.find(cur_colref_to_push);
|
|
114060
|
+
D_ASSERT(entry != map_col_conjunctions.end());
|
|
114061
|
+
auto &conjunctions_to_push = entry->second.back();
|
|
114062
|
+
|
|
114063
|
+
if (!cur_colref_to_push->Equals(column_ref)) {
|
|
114064
|
+
// check for multiple colunms in the same root OR node
|
|
114065
|
+
if (cur_root_or == cur_conjunction) {
|
|
114066
|
+
return false;
|
|
114067
|
+
}
|
|
114068
|
+
// found an AND using a different column, we should stop the look up
|
|
114069
|
+
if (cur_conjunction->GetExpressionType() == ExpressionType::CONJUNCTION_AND) {
|
|
114070
|
+
return false;
|
|
114071
|
+
}
|
|
114072
|
+
|
|
114073
|
+
// found a different column, AND conditions cannot be preserved anymore
|
|
114074
|
+
conjunctions_to_push->preserve_and = false;
|
|
114075
|
+
return true;
|
|
114076
|
+
}
|
|
114077
|
+
|
|
114078
|
+
auto &last_conjunction = conjunctions_to_push->conjunctions.back();
|
|
114079
|
+
if (cur_conjunction->GetExpressionType() == last_conjunction->GetExpressionType()) {
|
|
114080
|
+
last_conjunction->children.emplace_back(comparison_expr->Copy());
|
|
114081
|
+
} else {
|
|
114082
|
+
auto new_conjunction = make_unique<BoundConjunctionExpression>(cur_conjunction->GetExpressionType());
|
|
114083
|
+
new_conjunction->children.emplace_back(comparison_expr->Copy());
|
|
114084
|
+
conjunctions_to_push->conjunctions.emplace_back(move(new_conjunction));
|
|
114085
|
+
}
|
|
114086
|
+
return true;
|
|
114087
|
+
}
|
|
114088
|
+
|
|
114089
|
+
void FilterCombiner::GenerateORFilters(TableFilterSet &table_filter, vector<idx_t> &column_ids) {
|
|
114090
|
+
for (const auto colref : vec_colref_insertion_order) {
|
|
114091
|
+
auto column_index = column_ids[colref->binding.column_index];
|
|
114092
|
+
if (column_index == COLUMN_IDENTIFIER_ROW_ID) {
|
|
114093
|
+
break;
|
|
114094
|
+
}
|
|
114095
|
+
|
|
114096
|
+
for (const auto &conjunctions_to_push : map_col_conjunctions[colref]) {
|
|
114097
|
+
// root OR filter to push into the TableFilter
|
|
114098
|
+
auto root_or_filter = make_unique<ConjunctionOrFilter>();
|
|
114099
|
+
// variable to hold the last conjuntion filter pointer
|
|
114100
|
+
// the next filter will be added into it, i.e., we create a chain of conjunction filters
|
|
114101
|
+
ConjunctionFilter *last_conj_filter = root_or_filter.get();
|
|
114102
|
+
|
|
114103
|
+
for (auto &conjunction : conjunctions_to_push->conjunctions) {
|
|
114104
|
+
if (conjunction->GetExpressionType() == ExpressionType::CONJUNCTION_AND &&
|
|
114105
|
+
conjunctions_to_push->preserve_and) {
|
|
114106
|
+
GenerateConjunctionFilter<ConjunctionAndFilter>(conjunction.get(), last_conj_filter);
|
|
114107
|
+
} else {
|
|
114108
|
+
GenerateConjunctionFilter<ConjunctionOrFilter>(conjunction.get(), last_conj_filter);
|
|
114109
|
+
}
|
|
114110
|
+
}
|
|
114111
|
+
table_filter.PushFilter(column_index, move(root_or_filter));
|
|
114112
|
+
}
|
|
114113
|
+
}
|
|
114114
|
+
map_col_conjunctions.clear();
|
|
114115
|
+
vec_colref_insertion_order.clear();
|
|
114116
|
+
}
|
|
114117
|
+
|
|
113870
114118
|
} // namespace duckdb
|
|
113871
114119
|
//===----------------------------------------------------------------------===//
|
|
113872
114120
|
// DuckDB
|
|
@@ -138758,7 +139006,8 @@ unique_ptr<CreateStatement> Transformer::TransformCreateFunction(duckdb_libpgque
|
|
|
138758
139006
|
if (macro_func->default_parameters.find(param->alias) != macro_func->default_parameters.end()) {
|
|
138759
139007
|
throw ParserException("Duplicate default parameter: '%s'", param->alias);
|
|
138760
139008
|
}
|
|
138761
|
-
|
|
139009
|
+
auto alias = param->alias;
|
|
139010
|
+
macro_func->default_parameters[alias] = move(param);
|
|
138762
139011
|
} else if (param->GetExpressionClass() == ExpressionClass::COLUMN_REF) {
|
|
138763
139012
|
// positional parameters
|
|
138764
139013
|
if (!macro_func->default_parameters.empty()) {
|
|
@@ -149818,7 +150067,7 @@ void ExpressionIterator::EnumerateQueryNodeChildren(BoundQueryNode &node,
|
|
|
149818
150067
|
|
|
149819
150068
|
namespace duckdb {
|
|
149820
150069
|
|
|
149821
|
-
ConjunctionOrFilter::ConjunctionOrFilter() :
|
|
150070
|
+
ConjunctionOrFilter::ConjunctionOrFilter() : ConjunctionFilter(TableFilterType::CONJUNCTION_OR) {
|
|
149822
150071
|
}
|
|
149823
150072
|
|
|
149824
150073
|
FilterPropagateResult ConjunctionOrFilter::CheckStatistics(BaseStatistics &stats) {
|
|
@@ -149847,7 +150096,7 @@ string ConjunctionOrFilter::ToString(const string &column_name) {
|
|
|
149847
150096
|
}
|
|
149848
150097
|
|
|
149849
150098
|
bool ConjunctionOrFilter::Equals(const TableFilter &other_p) const {
|
|
149850
|
-
if (!
|
|
150099
|
+
if (!ConjunctionFilter::Equals(other_p)) {
|
|
149851
150100
|
return false;
|
|
149852
150101
|
}
|
|
149853
150102
|
auto &other = (ConjunctionOrFilter &)other_p;
|
|
@@ -149862,11 +150111,11 @@ bool ConjunctionOrFilter::Equals(const TableFilter &other_p) const {
|
|
|
149862
150111
|
return true;
|
|
149863
150112
|
}
|
|
149864
150113
|
|
|
149865
|
-
ConjunctionAndFilter::ConjunctionAndFilter() :
|
|
150114
|
+
ConjunctionAndFilter::ConjunctionAndFilter() : ConjunctionFilter(TableFilterType::CONJUNCTION_AND) {
|
|
149866
150115
|
}
|
|
149867
150116
|
|
|
149868
150117
|
FilterPropagateResult ConjunctionAndFilter::CheckStatistics(BaseStatistics &stats) {
|
|
149869
|
-
// the
|
|
150118
|
+
// the AND filter is true if ALL of the children is true
|
|
149870
150119
|
D_ASSERT(!child_filters.empty());
|
|
149871
150120
|
auto result = FilterPropagateResult::FILTER_ALWAYS_TRUE;
|
|
149872
150121
|
for (auto &filter : child_filters) {
|
|
@@ -149892,7 +150141,7 @@ string ConjunctionAndFilter::ToString(const string &column_name) {
|
|
|
149892
150141
|
}
|
|
149893
150142
|
|
|
149894
150143
|
bool ConjunctionAndFilter::Equals(const TableFilter &other_p) const {
|
|
149895
|
-
if (!
|
|
150144
|
+
if (!ConjunctionFilter::Equals(other_p)) {
|
|
149896
150145
|
return false;
|
|
149897
150146
|
}
|
|
149898
150147
|
auto &other = (ConjunctionAndFilter &)other_p;
|
|
@@ -160109,6 +160358,14 @@ FilterPropagateResult NumericStatistics::CheckZonemap(ExpressionType comparison_
|
|
|
160109
160358
|
} else {
|
|
160110
160359
|
return FilterPropagateResult::FILTER_ALWAYS_FALSE;
|
|
160111
160360
|
}
|
|
160361
|
+
case ExpressionType::COMPARE_NOTEQUAL:
|
|
160362
|
+
if (constant < min || constant > max) {
|
|
160363
|
+
return FilterPropagateResult::FILTER_ALWAYS_TRUE;
|
|
160364
|
+
} else if (min == max && min == constant) {
|
|
160365
|
+
// corner case of a cluster with one numeric equal to the target constant
|
|
160366
|
+
return FilterPropagateResult::FILTER_ALWAYS_FALSE;
|
|
160367
|
+
}
|
|
160368
|
+
return FilterPropagateResult::NO_PRUNING_POSSIBLE;
|
|
160112
160369
|
case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
|
|
160113
160370
|
// X >= C
|
|
160114
160371
|
// this can be true only if max(X) >= C
|
|
@@ -160409,6 +160666,11 @@ FilterPropagateResult StringStatistics::CheckZonemap(ExpressionType comparison_t
|
|
|
160409
160666
|
} else {
|
|
160410
160667
|
return FilterPropagateResult::FILTER_ALWAYS_FALSE;
|
|
160411
160668
|
}
|
|
160669
|
+
case ExpressionType::COMPARE_NOTEQUAL:
|
|
160670
|
+
if (min_comp < 0 || max_comp > 0) {
|
|
160671
|
+
return FilterPropagateResult::FILTER_ALWAYS_TRUE;
|
|
160672
|
+
}
|
|
160673
|
+
return FilterPropagateResult::NO_PRUNING_POSSIBLE;
|
|
160412
160674
|
case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
|
|
160413
160675
|
case ExpressionType::COMPARE_GREATERTHAN:
|
|
160414
160676
|
if (max_comp <= 0) {
|
|
@@ -162532,6 +162794,16 @@ static void FilterSelectionSwitch(T *vec, T *predicate, SelectionVector &sel, id
|
|
|
162532
162794
|
}
|
|
162533
162795
|
break;
|
|
162534
162796
|
}
|
|
162797
|
+
case ExpressionType::COMPARE_NOTEQUAL: {
|
|
162798
|
+
if (mask.AllValid()) {
|
|
162799
|
+
approved_tuple_count =
|
|
162800
|
+
TemplatedFilterSelection<T, NotEquals, false>(vec, predicate, sel, approved_tuple_count, mask, new_sel);
|
|
162801
|
+
} else {
|
|
162802
|
+
approved_tuple_count =
|
|
162803
|
+
TemplatedFilterSelection<T, NotEquals, true>(vec, predicate, sel, approved_tuple_count, mask, new_sel);
|
|
162804
|
+
}
|
|
162805
|
+
break;
|
|
162806
|
+
}
|
|
162535
162807
|
case ExpressionType::COMPARE_LESSTHAN: {
|
|
162536
162808
|
if (mask.AllValid()) {
|
|
162537
162809
|
approved_tuple_count =
|
|
@@ -162601,15 +162873,44 @@ static idx_t TemplatedNullSelection(SelectionVector &sel, idx_t approved_tuple_c
|
|
|
162601
162873
|
}
|
|
162602
162874
|
}
|
|
162603
162875
|
|
|
162604
|
-
|
|
162605
|
-
|
|
162876
|
+
idx_t ColumnSegment::FilterSelection(SelectionVector &sel, Vector &result, const TableFilter &filter,
|
|
162877
|
+
idx_t &approved_tuple_count, ValidityMask &mask) {
|
|
162606
162878
|
switch (filter.filter_type) {
|
|
162879
|
+
case TableFilterType::CONJUNCTION_OR: {
|
|
162880
|
+
// similar to the CONJUNCTION_AND, but we need to take care of the SelectionVectors (OR all of them)
|
|
162881
|
+
idx_t count_total = 0;
|
|
162882
|
+
SelectionVector result_sel(approved_tuple_count);
|
|
162883
|
+
auto &conjunction_or = (ConjunctionOrFilter &)filter;
|
|
162884
|
+
for (auto &child_filter : conjunction_or.child_filters) {
|
|
162885
|
+
SelectionVector temp_sel;
|
|
162886
|
+
temp_sel.Initialize(sel);
|
|
162887
|
+
idx_t temp_tuple_count = approved_tuple_count;
|
|
162888
|
+
idx_t temp_count = FilterSelection(temp_sel, result, *child_filter, temp_tuple_count, mask);
|
|
162889
|
+
// tuples passed, move them into the actual result vector
|
|
162890
|
+
for (idx_t i = 0; i < temp_count; i++) {
|
|
162891
|
+
auto new_idx = temp_sel.get_index(i);
|
|
162892
|
+
bool is_new_idx = true;
|
|
162893
|
+
for (idx_t res_idx = 0; res_idx < count_total; res_idx++) {
|
|
162894
|
+
if (result_sel.get_index(res_idx) == new_idx) {
|
|
162895
|
+
is_new_idx = false;
|
|
162896
|
+
break;
|
|
162897
|
+
}
|
|
162898
|
+
}
|
|
162899
|
+
if (is_new_idx) {
|
|
162900
|
+
result_sel.set_index(count_total++, new_idx);
|
|
162901
|
+
}
|
|
162902
|
+
}
|
|
162903
|
+
}
|
|
162904
|
+
sel.Initialize(result_sel);
|
|
162905
|
+
approved_tuple_count = count_total;
|
|
162906
|
+
return approved_tuple_count;
|
|
162907
|
+
}
|
|
162607
162908
|
case TableFilterType::CONJUNCTION_AND: {
|
|
162608
162909
|
auto &conjunction_and = (ConjunctionAndFilter &)filter;
|
|
162609
162910
|
for (auto &child_filter : conjunction_and.child_filters) {
|
|
162610
162911
|
FilterSelection(sel, result, *child_filter, approved_tuple_count, mask);
|
|
162611
162912
|
}
|
|
162612
|
-
|
|
162913
|
+
return approved_tuple_count;
|
|
162613
162914
|
}
|
|
162614
162915
|
case TableFilterType::CONSTANT_COMPARISON: {
|
|
162615
162916
|
auto &constant_filter = (ConstantFilter &)filter;
|
|
@@ -162722,14 +163023,12 @@ void ColumnSegment::FilterSelection(SelectionVector &sel, Vector &result, const
|
|
|
162722
163023
|
default:
|
|
162723
163024
|
throw InvalidTypeException(result.GetType(), "Invalid type for filter pushed down to table comparison");
|
|
162724
163025
|
}
|
|
162725
|
-
|
|
163026
|
+
return approved_tuple_count;
|
|
162726
163027
|
}
|
|
162727
163028
|
case TableFilterType::IS_NULL:
|
|
162728
|
-
TemplatedNullSelection<true>(sel, approved_tuple_count, mask);
|
|
162729
|
-
break;
|
|
163029
|
+
return TemplatedNullSelection<true>(sel, approved_tuple_count, mask);
|
|
162730
163030
|
case TableFilterType::IS_NOT_NULL:
|
|
162731
|
-
TemplatedNullSelection<false>(sel, approved_tuple_count, mask);
|
|
162732
|
-
break;
|
|
163031
|
+
return TemplatedNullSelection<false>(sel, approved_tuple_count, mask);
|
|
162733
163032
|
default:
|
|
162734
163033
|
throw InternalException("FIXME: unsupported type for filter selection");
|
|
162735
163034
|
}
|
package/src/duckdb.hpp
CHANGED
|
@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
|
|
|
11
11
|
#pragma once
|
|
12
12
|
#define DUCKDB_AMALGAMATION 1
|
|
13
13
|
#define DUCKDB_AMALGAMATION_EXTENDED 1
|
|
14
|
-
#define DUCKDB_SOURCE_ID "
|
|
15
|
-
#define DUCKDB_VERSION "0.3.2-
|
|
14
|
+
#define DUCKDB_SOURCE_ID "684aaf4e7"
|
|
15
|
+
#define DUCKDB_VERSION "0.3.2-dev816"
|
|
16
16
|
//===----------------------------------------------------------------------===//
|
|
17
17
|
// DuckDB
|
|
18
18
|
//
|
|
@@ -1097,6 +1097,11 @@ struct StructType {
|
|
|
1097
1097
|
DUCKDB_API static idx_t GetChildCount(const LogicalType &type);
|
|
1098
1098
|
};
|
|
1099
1099
|
|
|
1100
|
+
struct MapType {
|
|
1101
|
+
DUCKDB_API static const LogicalType &KeyType(const LogicalType &type);
|
|
1102
|
+
DUCKDB_API static const LogicalType &ValueType(const LogicalType &type);
|
|
1103
|
+
};
|
|
1104
|
+
|
|
1100
1105
|
|
|
1101
1106
|
string LogicalTypeIdToString(LogicalTypeId type);
|
|
1102
1107
|
|
|
@@ -19200,8 +19205,8 @@ public:
|
|
|
19200
19205
|
//! Fetch a value of the specific row id and append it to the result
|
|
19201
19206
|
void FetchRow(ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx);
|
|
19202
19207
|
|
|
19203
|
-
static
|
|
19204
|
-
|
|
19208
|
+
static idx_t FilterSelection(SelectionVector &sel, Vector &result, const TableFilter &filter,
|
|
19209
|
+
idx_t &approved_tuple_count, ValidityMask &mask);
|
|
19205
19210
|
|
|
19206
19211
|
//! Skip a scan forward to the row_index specified in the scan state
|
|
19207
19212
|
void Skip(ColumnScanState &state);
|
|
@@ -20102,27 +20107,40 @@ void NumericStatistics::Update<list_entry_t>(SegmentStatistics &stats, list_entr
|
|
|
20102
20107
|
|
|
20103
20108
|
|
|
20104
20109
|
namespace duckdb {
|
|
20105
|
-
|
|
20106
|
-
class ConjunctionOrFilter : public TableFilter {
|
|
20110
|
+
class ConjunctionFilter : public TableFilter {
|
|
20107
20111
|
public:
|
|
20108
|
-
|
|
20112
|
+
ConjunctionFilter(TableFilterType filter_type_p) : TableFilter(filter_type_p) {
|
|
20113
|
+
}
|
|
20109
20114
|
|
|
20110
|
-
|
|
20115
|
+
virtual ~ConjunctionFilter() {
|
|
20116
|
+
}
|
|
20117
|
+
|
|
20118
|
+
//! The filters of this conjunction
|
|
20111
20119
|
vector<unique_ptr<TableFilter>> child_filters;
|
|
20112
20120
|
|
|
20121
|
+
public:
|
|
20122
|
+
virtual FilterPropagateResult CheckStatistics(BaseStatistics &stats) = 0;
|
|
20123
|
+
virtual string ToString(const string &column_name) = 0;
|
|
20124
|
+
|
|
20125
|
+
virtual bool Equals(const TableFilter &other) const {
|
|
20126
|
+
return TableFilter::Equals(other);
|
|
20127
|
+
}
|
|
20128
|
+
};
|
|
20129
|
+
|
|
20130
|
+
class ConjunctionOrFilter : public ConjunctionFilter {
|
|
20131
|
+
public:
|
|
20132
|
+
ConjunctionOrFilter();
|
|
20133
|
+
|
|
20113
20134
|
public:
|
|
20114
20135
|
FilterPropagateResult CheckStatistics(BaseStatistics &stats) override;
|
|
20115
20136
|
string ToString(const string &column_name) override;
|
|
20116
20137
|
bool Equals(const TableFilter &other) const override;
|
|
20117
20138
|
};
|
|
20118
20139
|
|
|
20119
|
-
class ConjunctionAndFilter : public
|
|
20140
|
+
class ConjunctionAndFilter : public ConjunctionFilter {
|
|
20120
20141
|
public:
|
|
20121
20142
|
ConjunctionAndFilter();
|
|
20122
20143
|
|
|
20123
|
-
//! The filters to OR together
|
|
20124
|
-
vector<unique_ptr<TableFilter>> child_filters;
|
|
20125
|
-
|
|
20126
20144
|
public:
|
|
20127
20145
|
FilterPropagateResult CheckStatistics(BaseStatistics &stats) override;
|
|
20128
20146
|
string ToString(const string &column_name) override;
|