duckdb 0.5.2-dev815.0 → 0.5.2-dev841.0

package/src/duckdb.cpp

@@ -29106,7 +29106,7 @@ struct RowOperations {
 namespace duckdb {
 
 template <class OP, class RETURN_TYPE, typename... ARGS>
-RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&... args) {
+RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&...args) {
 	D_ASSERT(radix_bits <= sizeof(hash_t) * 8);
 	switch (radix_bits) {
 	case 1:
@@ -29135,7 +29135,7 @@ RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&... args) {
 }
 
 template <class OP, class RETURN_TYPE, idx_t radix_bits_1, typename... ARGS>
-RETURN_TYPE DoubleRadixBitsSwitch2(idx_t radix_bits_2, ARGS &&... args) {
+RETURN_TYPE DoubleRadixBitsSwitch2(idx_t radix_bits_2, ARGS &&...args) {
 	D_ASSERT(radix_bits_2 <= sizeof(hash_t) * 8);
 	switch (radix_bits_2) {
 	case 1:
@@ -29164,7 +29164,7 @@ RETURN_TYPE DoubleRadixBitsSwitch2(idx_t radix_bits_2, ARGS &&... args) {
 }
 
 template <class OP, class RETURN_TYPE, typename... ARGS>
-RETURN_TYPE DoubleRadixBitsSwitch1(idx_t radix_bits_1, idx_t radix_bits_2, ARGS &&... args) {
+RETURN_TYPE DoubleRadixBitsSwitch1(idx_t radix_bits_1, idx_t radix_bits_2, ARGS &&...args) {
 	D_ASSERT(radix_bits_1 <= sizeof(hash_t) * 8);
 	switch (radix_bits_1) {
 	case 1:
@@ -49455,6 +49455,9 @@ void Vector::Initialize(bool zero_data, idx_t capacity) {
 			memset(data, 0, capacity * type_size);
 		}
 	}
+	if (capacity > STANDARD_VECTOR_SIZE) {
+		validity.Resize(STANDARD_VECTOR_SIZE, capacity);
+	}
 }
 
 struct DataArrays {
@@ -93661,6 +93664,13 @@ unique_ptr<FunctionData> HistogramBindFunction(ClientContext &context, Aggregate
                                                vector<unique_ptr<Expression>> &arguments) {
 
 	D_ASSERT(arguments.size() == 1);
+
+	if (arguments[0]->return_type.id() == LogicalTypeId::LIST ||
+	    arguments[0]->return_type.id() == LogicalTypeId::STRUCT ||
+	    arguments[0]->return_type.id() == LogicalTypeId::MAP) {
+		throw NotImplementedException("Unimplemented type for histogram %s", arguments[0]->return_type.ToString());
+	}
+
 	child_list_t<LogicalType> struct_children;
 	struct_children.push_back({"key", LogicalType::LIST(arguments[0]->return_type)});
 	struct_children.push_back({"value", LogicalType::LIST(LogicalType::UBIGINT)});
@@ -95714,6 +95724,11 @@ struct VectorCastHelpers {
 	}
 };
 
+struct VectorStringifiedListParser {
+	static idx_t CountParts(const string_t &input);
+	static bool SplitStringifiedList(const string_t &input, string_t *child_data, idx_t &child_start, Vector &child);
+};
+
 } // namespace duckdb
 
 
@@ -96484,20 +96499,8 @@ BoundCastInfo DefaultCasts::EnumCastSwitch(BindCastInput &input, const LogicalTy
 
 namespace duckdb {
 
-struct ListBoundCastData : public BoundCastData {
-	explicit ListBoundCastData(BoundCastInfo child_cast) : child_cast_info(move(child_cast)) {
-	}
-
-	BoundCastInfo child_cast_info;
-
-public:
-	unique_ptr<BoundCastData> Copy() const override {
-		return make_unique<ListBoundCastData>(child_cast_info.Copy());
-	}
-};
-
-unique_ptr<BoundCastData> BindListToListCast(BindCastInput &input, const LogicalType &source,
-                                             const LogicalType &target) {
+unique_ptr<BoundCastData> ListBoundCastData::BindListToListCast(BindCastInput &input, const LogicalType &source,
+                                                                const LogicalType &target) {
 	vector<BoundCastInfo> child_cast_info;
 	auto &source_child_type = ListType::GetChildType(source);
 	auto &result_child_type = ListType::GetChildType(target);
@@ -96608,11 +96611,11 @@ static bool ListToVarcharCast(Vector &source, Vector &result, idx_t count, CastP
 BoundCastInfo DefaultCasts::ListCastSwitch(BindCastInput &input, const LogicalType &source, const LogicalType &target) {
 	switch (target.id()) {
 	case LogicalTypeId::LIST:
-		return BoundCastInfo(ListToListCast, BindListToListCast(input, source, target));
+		return BoundCastInfo(ListToListCast, ListBoundCastData::BindListToListCast(input, source, target));
 	case LogicalTypeId::VARCHAR:
 	case LogicalTypeId::JSON:
-		return BoundCastInfo(ListToVarcharCast,
-		                     BindListToListCast(input, source, LogicalType::LIST(LogicalType::VARCHAR)));
+		return BoundCastInfo(ListToVarcharCast, ListBoundCastData::BindListToListCast(
+		                                            input, source, LogicalType::LIST(LogicalType::VARCHAR)));
 	default:
 		return DefaultCasts::TryVectorNullCast;
 	}
@@ -96950,9 +96953,97 @@ static BoundCastInfo VectorStringCastNumericSwitch(BindCastInput &input, const L
 	}
 }
 
+bool StringListCastLoop(string_t *source_data, ValidityMask &source_mask, Vector &result, ValidityMask &result_mask,
+                        idx_t count, CastParameters &parameters, const SelectionVector *sel) {
+
+	idx_t total_list_size = 0;
+	for (idx_t i = 0; i < count; i++) {
+		idx_t idx = i;
+		if (sel) {
+			idx = sel->get_index(i);
+		}
+		if (!source_mask.RowIsValid(idx)) {
+			continue;
+		}
+		total_list_size += VectorStringifiedListParser::CountParts(source_data[idx]);
+	}
+
+	Vector varchar_vector(LogicalType::VARCHAR, total_list_size);
+
+	ListVector::Reserve(result, total_list_size);
+	ListVector::SetListSize(result, total_list_size);
+
+	auto list_data = ListVector::GetData(result);
+	auto child_data = FlatVector::GetData<string_t>(varchar_vector);
+
+	bool all_converted = true;
+	idx_t total = 0;
+	for (idx_t i = 0; i < count; i++) {
+		idx_t idx = i;
+		if (sel) {
+			idx = sel->get_index(i);
+		}
+		if (!source_mask.RowIsValid(idx)) {
+			result_mask.SetInvalid(i);
+			continue;
+		}
+
+		list_data[i].offset = total;
+		auto valid =
+		    VectorStringifiedListParser::SplitStringifiedList(source_data[idx], child_data, total, varchar_vector);
+		if (!valid) {
+			string text = "Type VARCHAR with value '" + source_data[idx].GetString() +
+			              "' can't be cast to the destination type LIST";
+			HandleVectorCastError::Operation<string_t>(text, result_mask, idx, parameters.error_message, all_converted);
+		}
+		list_data[i].length = total - list_data[i].offset; // length is the amount of parts coming from this string
+	}
+	D_ASSERT(total_list_size == total);
+
+	auto &result_child = ListVector::GetEntry(result);
+	auto &cast_data = (ListBoundCastData &)*parameters.cast_data;
+	CastParameters child_parameters(parameters, cast_data.child_cast_info.cast_data.get());
+	return cast_data.child_cast_info.function(varchar_vector, result_child, total_list_size, child_parameters) &&
+	       all_converted;
+}
+
+bool StringListCast(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
+	D_ASSERT(source.GetType().id() == LogicalTypeId::VARCHAR);
+	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
+
+	switch (source.GetVectorType()) {
+	case VectorType::CONSTANT_VECTOR: {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+
+		auto source_data = ConstantVector::GetData<string_t>(source);
+		auto &source_mask = ConstantVector::Validity(source);
+		auto &result_mask = ConstantVector::Validity(result);
+
+		return StringListCastLoop(source_data, source_mask, result, result_mask, 1, parameters, nullptr);
+	}
+	default: {
+		UnifiedVectorFormat unified_source;
+		result.SetVectorType(VectorType::FLAT_VECTOR);
+
+		source.ToUnifiedFormat(count, unified_source);
+		auto source_sel = unified_source.sel;
+		auto source_data = (string_t *)unified_source.data;
+		auto &source_mask = unified_source.validity;
+		auto &result_mask = FlatVector::Validity(result);
+
+		return StringListCastLoop(source_data, source_mask, result, result_mask, count, parameters, source_sel);
+	}
+	}
+}
+
+BoundCastInfo StringToListCast(BindCastInput &input, const LogicalType &source, const LogicalType &target) {
+	// second argument allows for a secondary casting function to be passed in the CastParameters
+	return BoundCastInfo(&StringListCast,
+	                     ListBoundCastData::BindListToListCast(input, LogicalType::LIST(LogicalType::VARCHAR), target));
+}
+
 BoundCastInfo DefaultCasts::StringCastSwitch(BindCastInput &input, const LogicalType &source,
                                              const LogicalType &target) {
-	// now switch on the target type
 	switch (target.id()) {
 	case LogicalTypeId::DATE:
 		return BoundCastInfo(&VectorCastHelpers::TryCastErrorLoop<string_t, date_t, duckdb::TryCastErrorMessage>);
@@ -96980,6 +97071,8 @@ BoundCastInfo DefaultCasts::StringCastSwitch(BindCastInput &input, const Logical
 	case LogicalTypeId::VARCHAR:
 	case LogicalTypeId::JSON:
 		return &DefaultCasts::ReinterpretCast;
+	case LogicalTypeId::LIST:
+		return StringToListCast(input, source, target);
 	default:
 		return VectorStringCastNumericSwitch(input, source, target);
 	}
@@ -97341,6 +97434,144 @@ BoundCastInfo DefaultCasts::UUIDCastSwitch(BindCastInput &input, const LogicalTy
 } // namespace duckdb
 
 
+namespace duckdb {
+
+struct CountPartOperation {
+	idx_t count = 0;
+
+	void HandleValue(const char *buf, idx_t start_pos, idx_t pos) {
+		count++;
+	}
+};
+
+struct SplitStringOperation {
+	SplitStringOperation(string_t *child_data, idx_t &child_start, Vector &child)
+	    : child_data(child_data), child_start(child_start), child(child) {
+	}
+
+	string_t *child_data;
+	idx_t &child_start;
+	Vector &child;
+
+	void HandleValue(const char *buf, idx_t start_pos, idx_t pos) {
+
+		if ((pos - start_pos) >= 4 && buf[start_pos] == 'N' && buf[start_pos + 1] == 'U' && buf[start_pos + 2] == 'L' &&
+		    buf[start_pos + 3] == 'L') {
+			FlatVector::SetNull(child, child_start, true);
+			child_start++;
+			return;
+		}
+		child_data[child_start] = StringVector::AddString(child, buf + start_pos, pos - start_pos);
+		child_start++;
+	}
+};
+
+static bool SkipToCloseQuotes(idx_t &pos, const char *buf, idx_t &len) {
+	char quote = buf[pos];
+	pos++;
+
+	while (pos < len) {
+		if (buf[pos] == quote) {
+			return true;
+		}
+		pos++;
+	}
+	return false;
+}
+
+static bool SkipToClose(idx_t &idx, const char *buf, idx_t &len, idx_t &lvl) {
+	while (idx < len) {
+		if (buf[idx] == '[') {
+			if (!SkipToClose(++idx, buf, len, lvl)) {
+				return false;
+			}
+			lvl++;
+			idx++;
+		}
+		if (buf[idx] == '"' || buf[idx] == '\'') {
+			SkipToCloseQuotes(idx, buf, len);
+		}
+		if (buf[idx] == ']') {
+			lvl--;
+			return true;
+		}
+		idx++;
+	}
+	return false;
+}
+
+template <class OP>
+static bool SplitStringifiedListInternal(const string_t &input, OP &state) {
+	const char *buf = input.GetDataUnsafe();
+	idx_t len = input.GetSize();
+	idx_t lvl = 1;
+	idx_t pos = 0;
+
+	while (pos < len && StringUtil::CharacterIsSpace(buf[pos])) {
+		pos++;
+	}
+	if (pos == len || buf[pos] != '[') {
+		return false;
+	}
+	pos++;
+	while (pos < len && StringUtil::CharacterIsSpace(buf[pos])) {
+		pos++;
+	}
+
+	idx_t start_pos = pos;
+	while (pos < len) {
+		if (buf[pos] == '[') {
+			if (!SkipToClose(++pos, buf, len, ++lvl)) {
+				return false;
+			}
+		} else if (buf[pos] == '"' || buf[pos] == '\'') {
+			SkipToCloseQuotes(pos, buf, len);
+		} else if (buf[pos] == ',' || buf[pos] == ']') {
+			idx_t trailing_whitespace = 0;
+			while (StringUtil::CharacterIsSpace(buf[pos - trailing_whitespace - 1])) {
+				trailing_whitespace++;
+			}
+			if (!(buf[pos] == ']' && start_pos == (pos))) {
+				state.HandleValue(buf, start_pos, pos - trailing_whitespace);
+			} // else the list is empty
+			if (buf[pos] == ']') {
+				lvl--;
+				break;
+			}
+			while (pos + 1 < len && StringUtil::CharacterIsSpace(buf[pos + 1])) {
+				pos++;
+			}
+			start_pos = pos + 1;
+		}
+		pos++;
+	}
+	pos++;
+	while (pos < len) {
+		if (!StringUtil::CharacterIsSpace(buf[pos])) {
+			return false;
+		}
+		pos++;
+	}
+	if (lvl != 0) {
+		return false;
+	}
+	return true;
+}
+
+bool VectorStringifiedListParser::SplitStringifiedList(const string_t &input, string_t *child_data, idx_t &child_start,
+                                                       Vector &child) {
+	SplitStringOperation state(child_data, child_start, child);
+	return SplitStringifiedListInternal<SplitStringOperation>(input, state);
+}
+
+idx_t VectorStringifiedListParser::CountParts(const string_t &input) {
+	CountPartOperation state;
+	SplitStringifiedListInternal<CountPartOperation>(input, state);
+	return state.count;
+}
+} // namespace duckdb
+
+
 namespace duckdb {
 
 //! The target type determines the preferred implicit casts
@@ -146173,12 +146404,11 @@ private:
 	//! Find Joins with a DelimGet that can be removed
 	void FindCandidates(unique_ptr<LogicalOperator> *op_ptr, vector<unique_ptr<LogicalOperator> *> &candidates);
 	//! Try to remove a Join with a DelimGet, returns true if it was successful
-	bool RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, DeliminatorPlanUpdater &updater);
-	//! Replace references to a removed DelimGet, remove DelimJoins if all their DelimGets are gone
-	void UpdatePlan(LogicalOperator &op, expression_map_t<Expression *> &expr_map,
-	                column_binding_map_t<bool> &projection_map);
-	//! Whether the operator has one or more children of type DELIM_GET
-	bool HasChildDelimGet(LogicalOperator &op);
+	bool RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
+	                     DeliminatorPlanUpdater &updater);
+	//! Try to remove an inequality Join with a DelimGet, returns true if it was successful
+	bool RemoveInequalityCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
+	                               DeliminatorPlanUpdater &updater);
 };
 
 } // namespace duckdb
@@ -146202,31 +146432,64 @@ public:
 	//! Update the plan after a DelimGet has been removed
 	void VisitOperator(LogicalOperator &op) override;
 	void VisitExpression(unique_ptr<Expression> *expression) override;
-	//! Whether the operator has one or more children of type DELIM_GET
-	bool HasChildDelimGet(LogicalOperator &op);
+
+public:
 	expression_map_t<Expression *> expr_map;
 	column_binding_map_t<bool> projection_map;
+	column_binding_map_t<Expression *> reverse_proj_or_agg_map;
 	unique_ptr<LogicalOperator> temp_ptr;
 };
 
+static idx_t DelimGetCount(LogicalOperator &op) {
+	if (op.type == LogicalOperatorType::LOGICAL_DELIM_GET) {
+		return 1;
+	}
+	idx_t child_count = 0;
+	for (auto &child : op.children) {
+		child_count += DelimGetCount(*child);
+	}
+	return child_count;
+}
+
+static bool IsEqualityJoinCondition(JoinCondition &cond) {
+	switch (cond.comparison) {
+	case ExpressionType::COMPARE_EQUAL:
+	case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool InequalityDelimJoinCanBeEliminated(JoinType &join_type) {
+	switch (join_type) {
+	case JoinType::ANTI:
+	case JoinType::MARK:
+	case JoinType::SEMI:
+	case JoinType::SINGLE:
+		return true;
+	default:
+		return false;
+	}
+}
+
 void DeliminatorPlanUpdater::VisitOperator(LogicalOperator &op) {
 	VisitOperatorChildren(op);
 	VisitOperatorExpressions(op);
-	if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN && !HasChildDelimGet(op)) {
+	if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN && DelimGetCount(op) == 0) {
 		auto &delim_join = (LogicalDelimJoin &)op;
 		auto decs = &delim_join.duplicate_eliminated_columns;
 		for (auto &cond : delim_join.conditions) {
-			if (cond.comparison != ExpressionType::COMPARE_EQUAL &&
-			    cond.comparison != ExpressionType::COMPARE_NOT_DISTINCT_FROM) {
+			if (!IsEqualityJoinCondition(cond)) {
 				continue;
 			}
-			Expression *rhs = cond.right.get();
+			auto rhs = cond.right.get();
 			while (rhs->type == ExpressionType::OPERATOR_CAST) {
 				auto &cast = (BoundCastExpression &)*rhs;
 				rhs = cast.child.get();
 			}
 			if (rhs->type != ExpressionType::BOUND_COLUMN_REF) {
-				throw InternalException("Erorr in deliminator: expected a bound column reference");
+				throw InternalException("Error in Deliminator: expected a bound column reference");
 			}
 			auto &colref = (BoundColumnRefExpression &)*rhs;
 			if (projection_map.find(colref.binding) != projection_map.end()) {
@@ -146257,25 +146520,13 @@ void DeliminatorPlanUpdater::VisitExpression(unique_ptr<Expression> *expression)
 	}
 }
 
-bool DeliminatorPlanUpdater::HasChildDelimGet(LogicalOperator &op) {
-	if (op.type == LogicalOperatorType::LOGICAL_DELIM_GET) {
-		return true;
-	}
-	for (auto &child : op.children) {
-		if (HasChildDelimGet(*child)) {
-			return true;
-		}
-	}
-	return false;
-}
-
 unique_ptr<LogicalOperator> Deliminator::Optimize(unique_ptr<LogicalOperator> op) {
 	vector<unique_ptr<LogicalOperator> *> candidates;
 	FindCandidates(&op, candidates);
 
-	for (auto candidate : candidates) {
+	for (auto &candidate : candidates) {
 		DeliminatorPlanUpdater updater;
-		if (RemoveCandidate(candidate, updater)) {
+		if (RemoveCandidate(&op, candidate, updater)) {
 			updater.VisitOperator(*op);
 		}
 	}
@@ -146330,10 +146581,21 @@ static bool OperatorIsDelimGet(LogicalOperator &op) {
 	return false;
 }
 
-bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, DeliminatorPlanUpdater &updater) {
-	auto &proj_or_agg = **op_ptr;
+static bool ChildJoinTypeCanBeDeliminated(JoinType &join_type) {
+	switch (join_type) {
+	case JoinType::INNER:
+	case JoinType::SEMI:
+		return true;
+	default:
+		return false;
+	}
+}
+
+bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
+                                  DeliminatorPlanUpdater &updater) {
+	auto &proj_or_agg = **candidate;
 	auto &join = (LogicalComparisonJoin &)*proj_or_agg.children[0];
-	if (join.join_type != JoinType::INNER && join.join_type != JoinType::SEMI) {
+	if (!ChildJoinTypeCanBeDeliminated(join.join_type)) {
 		return false;
 	}
 
@@ -146351,13 +146613,10 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, Deliminat
 		return false;
 	}
 	// check if joining with the DelimGet is redundant, and collect relevant column information
+	bool all_equality_conditions = true;
 	vector<Expression *> nulls_are_not_equal_exprs;
 	for (auto &cond : join.conditions) {
-		if (cond.comparison != ExpressionType::COMPARE_EQUAL &&
-		    cond.comparison != ExpressionType::COMPARE_NOT_DISTINCT_FROM) {
-			// non-equality join condition
-			return false;
-		}
+		all_equality_conditions = all_equality_conditions && IsEqualityJoinCondition(cond);
 		auto delim_side = delim_idx == 0 ? cond.left.get() : cond.right.get();
 		auto other_side = delim_idx == 0 ? cond.right.get() : cond.left.get();
 		if (delim_side->type != ExpressionType::BOUND_COLUMN_REF) {
@@ -146370,10 +146629,12 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, Deliminat
 			nulls_are_not_equal_exprs.push_back(other_side);
 		}
 	}
+
 	// removed DelimGet columns are assigned a new ColumnBinding by Projection/Aggregation, keep track here
 	if (proj_or_agg.type == LogicalOperatorType::LOGICAL_PROJECTION) {
 		for (auto &cb : proj_or_agg.GetColumnBindings()) {
 			updater.projection_map[cb] = true;
+			updater.reverse_proj_or_agg_map[cb] = proj_or_agg.expressions[cb.column_index].get();
 			for (auto &expr : nulls_are_not_equal_exprs) {
 				if (proj_or_agg.expressions[cb.column_index]->Equals(expr)) {
 					updater.projection_map[cb] = false;
@@ -146383,8 +146644,19 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, Deliminat
 		}
 	} else {
 		auto &agg = (LogicalAggregate &)proj_or_agg;
+
+		// Create a vector of all exprs in the agg
+		vector<Expression *> all_agg_exprs;
+		for (auto &expr : agg.groups) {
+			all_agg_exprs.push_back(expr.get());
+		}
+		for (auto &expr : agg.expressions) {
+			all_agg_exprs.push_back(expr.get());
+		}
+
 		for (auto &cb : agg.GetColumnBindings()) {
 			updater.projection_map[cb] = true;
+			updater.reverse_proj_or_agg_map[cb] = all_agg_exprs[cb.column_index];
 			for (auto &expr : nulls_are_not_equal_exprs) {
 				if ((cb.table_index == agg.group_index && agg.groups[cb.column_index]->Equals(expr)) ||
 				    (cb.table_index == agg.aggregate_index && agg.expressions[cb.column_index]->Equals(expr))) {
@@ -146394,6 +146666,14 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, Deliminat
 			}
 		}
 	}
+
+	if (!all_equality_conditions) {
+		// we can get rid of an inequality join with a DelimGet, but only under specific circumstances
+		if (!RemoveInequalityCandidate(plan, candidate, updater)) {
+			return false;
+		}
+	}
+
 	// make a filter if needed
 	if (!nulls_are_not_equal_exprs.empty() || filter != nullptr) {
 		auto filter_op = make_unique<LogicalFilter>();
@@ -146421,6 +146701,146 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, Deliminat
 	return true;
 }
 
+static void GetDelimJoins(LogicalOperator &op, vector<LogicalOperator *> &delim_joins) {
+	for (auto &child : op.children) {
+		GetDelimJoins(*child, delim_joins);
+	}
+	if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
+		delim_joins.push_back(&op);
+	}
+}
+
+static bool HasChild(LogicalOperator *haystack, LogicalOperator *needle, idx_t &side) {
+	if (haystack == needle) {
+		return true;
+	}
+	for (idx_t i = 0; i < haystack->children.size(); i++) {
+		auto &child = haystack->children[i];
+		idx_t dummy_side;
+		if (HasChild(child.get(), needle, dummy_side)) {
+			side = i;
+			return true;
+		}
+	}
+	return false;
+}
+
+bool Deliminator::RemoveInequalityCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
+                                            DeliminatorPlanUpdater &updater) {
+	auto &proj_or_agg = **candidate;
+	// first, we find a DelimJoin in "plan" that has only one DelimGet as a child, which is in "candidate"
+	if (DelimGetCount(proj_or_agg) != 1) {
+		// the candidate therefore must have only a single DelimGet in its children
+		return false;
+	}
+
+	vector<LogicalOperator *> delim_joins;
+	GetDelimJoins(**plan, delim_joins);
+
+	LogicalOperator *parent = nullptr;
+	idx_t parent_delim_get_side;
+	for (auto dj : delim_joins) {
+		D_ASSERT(dj->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
+		if (!HasChild(dj, &proj_or_agg, parent_delim_get_side)) {
+			continue;
+		}
+		// we found a parent DelimJoin
+		if (DelimGetCount(*dj) != 1) {
+			// it has more than one DelimGet children
+			continue;
+		}
+
+		// we can only remove inequality join with a DelimGet if the parent DelimJoin has one of these join types
+		auto &delim_join = (LogicalDelimJoin &)*dj;
+		if (!InequalityDelimJoinCanBeEliminated(delim_join.join_type)) {
+			continue;
+		}
+
+		parent = dj;
+		break;
+	}
+	if (!parent) {
+		return false;
+	}
+
+	// we found the parent delim join, and we may be able to remove the child DelimGet join
+	// but we need to make sure that their conditions refer to exactly the same columns
+	auto &parent_delim_join = (LogicalDelimJoin &)*parent;
+	auto &join = (LogicalComparisonJoin &)*proj_or_agg.children[0];
+	if (parent_delim_join.conditions.size() != join.conditions.size()) {
+		// different number of conditions, can't replace
+		return false;
+	}
+
+	// we can only do this optimization under the following conditions:
+	// 1. all join expressions coming from the DelimGet side are colrefs
+	// 2. these expressions refer to colrefs coming from the proj/agg on top of the child DelimGet join
+	// 3. the expression (before it was proj/agg) can be found in the conditions of the child DelimGet join
+	for (auto &parent_cond : parent_delim_join.conditions) {
+		auto &parent_expr = parent_delim_get_side == 0 ? parent_cond.left : parent_cond.right;
+		if (parent_expr->type != ExpressionType::BOUND_COLUMN_REF) {
+			// can only deal with colrefs
+			return false;
+		}
+		auto &parent_colref = (BoundColumnRefExpression &)*parent_expr;
+		auto it = updater.reverse_proj_or_agg_map.find(parent_colref.binding);
+		if (it == updater.reverse_proj_or_agg_map.end()) {
+			// refers to a column that was not in the child DelimGet join
+			return false;
+		}
+		// try to find the corresponding child condition
+		// TODO: can be more flexible - allow CAST
+		auto child_expr = it->second;
+		bool found = false;
+		for (auto &child_cond : join.conditions) {
+			if (child_cond.left->Equals(child_expr) || child_cond.right->Equals(child_expr)) {
+				found = true;
+				break;
+			}
+		}
+		if (!found) {
+			// could not find the mapped expression in the child condition expressions
+			return false;
+		}
+	}
+
+	// TODO: we cannot perform the optimization here because our pure inequality joins don't implement
+	//  JoinType::SINGLE yet
+	if (parent_delim_join.join_type == JoinType::SINGLE) {
+		bool has_one_equality = false;
+		for (auto &cond : join.conditions) {
+			has_one_equality = has_one_equality || IsEqualityJoinCondition(cond);
+		}
+		if (!has_one_equality) {
+			return false;
+		}
+	}
+
+	// we are now sure that we can remove the child DelimGet join, so we basically do the same loop as above
+	// this time without checks because we already did them, and replace the expressions
+	for (auto &parent_cond : parent_delim_join.conditions) {
+		auto &parent_expr = parent_delim_get_side == 0 ? parent_cond.left : parent_cond.right;
+		auto &parent_colref = (BoundColumnRefExpression &)*parent_expr;
+		auto it = updater.reverse_proj_or_agg_map.find(parent_colref.binding);
+		auto child_expr = it->second;
+		for (auto &child_cond : join.conditions) {
+			if (!child_cond.left->Equals(child_expr) && !child_cond.right->Equals(child_expr)) {
+				continue;
+			}
+			parent_expr =
+			    make_unique<BoundColumnRefExpression>(parent_expr->alias, parent_expr->return_type, it->first);
+			parent_cond.comparison = child_cond.comparison;
+			break;
+		}
+	}
+
+	// no longer needs to be a delim join
+	parent_delim_join.duplicate_eliminated_columns.clear();
+	parent_delim_join.type = LogicalOperatorType::LOGICAL_COMPARISON_JOIN;
+
+	return true;
+}
+
 } // namespace duckdb