duckdb 0.8.2-dev2133.0 → 0.8.2-dev2208.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb/extension/json/json_functions.cpp +2 -1
- package/src/duckdb/extension/parquet/parquet_extension.cpp +2 -1
- package/src/duckdb/src/catalog/catalog.cpp +2 -6
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -4
- package/src/duckdb/src/catalog/catalog_entry/table_catalog_entry.cpp +0 -4
- package/src/duckdb/src/catalog/catalog_entry/type_catalog_entry.cpp +7 -26
- package/src/duckdb/src/catalog/catalog_set.cpp +0 -63
- package/src/duckdb/src/catalog/dependency_manager.cpp +0 -36
- package/src/duckdb/src/common/extra_type_info.cpp +24 -46
- package/src/duckdb/src/common/field_writer.cpp +0 -1
- package/src/duckdb/src/common/file_system.cpp +6 -6
- package/src/duckdb/src/common/filename_pattern.cpp +1 -1
- package/src/duckdb/src/common/gzip_file_system.cpp +7 -12
- package/src/duckdb/src/common/multi_file_reader.cpp +8 -5
- package/src/duckdb/src/common/serializer/buffered_file_reader.cpp +0 -9
- package/src/duckdb/src/common/types.cpp +12 -56
- package/src/duckdb/src/common/virtual_file_system.cpp +4 -0
- package/src/duckdb/src/core_functions/scalar/union/union_tag.cpp +1 -1
- package/src/duckdb/src/execution/column_binding_resolver.cpp +3 -7
- package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +1 -1
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +0 -10
- package/src/duckdb/src/execution/operator/schema/physical_attach.cpp +2 -1
- package/src/duckdb/src/execution/operator/schema/physical_create_type.cpp +2 -6
- package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +1 -2
- package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +14 -1
- package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +3 -17
- package/src/duckdb/src/execution/physical_plan/plan_delim_join.cpp +2 -5
- package/src/duckdb/src/execution/physical_plan_generator.cpp +1 -5
- package/src/duckdb/src/function/cast/string_cast.cpp +0 -1
- package/src/duckdb/src/function/scalar_function.cpp +3 -19
- package/src/duckdb/src/function/table/read_csv.cpp +2 -1
- package/src/duckdb/src/function/table/system/test_all_types.cpp +4 -4
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +0 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/type_catalog_entry.hpp +1 -4
- package/src/duckdb/src/include/duckdb/catalog/catalog_set.hpp +0 -6
- package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +2 -6
- package/src/duckdb/src/include/duckdb/common/field_writer.hpp +0 -4
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +8 -8
- package/src/duckdb/src/include/duckdb/common/filename_pattern.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +4 -0
- package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp +0 -4
- package/src/duckdb/src/include/duckdb/common/serializer.hpp +0 -7
- package/src/duckdb/src/include/duckdb/common/types/timestamp.hpp +14 -10
- package/src/duckdb/src/include/duckdb/common/types.hpp +5 -10
- package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +2 -0
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +0 -4
- package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -2
- package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +0 -3
- package/src/duckdb/src/include/duckdb/main/attached_database.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +1 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_create_index.hpp +10 -7
- package/src/duckdb/src/include/duckdb/planner/operator/logical_extension_operator.hpp +5 -0
- package/src/duckdb/src/include/duckdb/planner/operator_extension.hpp +1 -0
- package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp +0 -3
- package/src/duckdb/src/main/attached_database.cpp +2 -2
- package/src/duckdb/src/main/database.cpp +1 -1
- package/src/duckdb/src/main/db_instance_cache.cpp +14 -6
- package/src/duckdb/src/main/extension/extension_install.cpp +1 -1
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
- package/src/duckdb/src/optimizer/compressed_materialization.cpp +0 -1
- package/src/duckdb/src/optimizer/deliminator.cpp +7 -7
- package/src/duckdb/src/optimizer/unnest_rewriter.cpp +3 -5
- package/src/duckdb/src/parser/transform/helpers/transform_typename.cpp +16 -1
- package/src/duckdb/src/parser/transform/statement/transform_create_type.cpp +1 -1
- package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +3 -3
- package/src/duckdb/src/planner/binder/statement/bind_create.cpp +7 -23
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +0 -24
- package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +3 -5
- package/src/duckdb/src/planner/logical_operator.cpp +1 -6
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +3 -6
- package/src/duckdb/src/planner/operator/logical_comparison_join.cpp +2 -14
- package/src/duckdb/src/planner/operator/logical_create_index.cpp +21 -12
- package/src/duckdb/src/planner/operator/logical_extension_operator.cpp +24 -0
- package/src/duckdb/src/planner/planner.cpp +0 -1
- package/src/duckdb/src/storage/checkpoint_manager.cpp +1 -5
- package/src/duckdb/src/storage/meta_block_reader.cpp +0 -9
- package/src/duckdb/src/storage/serialization/serialize_create_info.cpp +2 -0
- package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +27 -0
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/wal_replay.cpp +1 -3
- package/src/duckdb/ub_src_planner_operator.cpp +0 -4
- package/src/statement.cpp +0 -2
- package/test/columns.test.ts +1 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_asof_join.hpp +0 -27
- package/src/duckdb/src/include/duckdb/planner/operator/logical_delim_join.hpp +0 -32
- package/src/duckdb/src/planner/operator/logical_asof_join.cpp +0 -14
- package/src/duckdb/src/planner/operator/logical_delim_join.cpp +0 -25
package/package.json
CHANGED
@@ -209,7 +209,8 @@ unique_ptr<TableRef> JSONFunctions::ReadJSONReplacement(ClientContext &context,
|
|
209
209
|
table_function->function = make_uniq<FunctionExpression>("read_json_auto", std::move(children));
|
210
210
|
|
211
211
|
if (!FileSystem::HasGlob(table_name)) {
|
212
|
-
|
212
|
+
auto &fs = FileSystem::GetFileSystem(context);
|
213
|
+
table_function->alias = fs.ExtractBaseName(table_name);
|
213
214
|
}
|
214
215
|
|
215
216
|
return std::move(table_function);
|
@@ -961,7 +961,8 @@ unique_ptr<TableRef> ParquetScanReplacement(ClientContext &context, const string
|
|
961
961
|
table_function->function = make_uniq<FunctionExpression>("parquet_scan", std::move(children));
|
962
962
|
|
963
963
|
if (!FileSystem::HasGlob(table_name)) {
|
964
|
-
|
964
|
+
auto &fs = FileSystem::GetFileSystem(context);
|
965
|
+
table_function->alias = fs.ExtractBaseName(table_name);
|
965
966
|
}
|
966
967
|
|
967
968
|
return std::move(table_function);
|
@@ -655,17 +655,13 @@ LogicalType Catalog::GetType(ClientContext &context, const string &schema, const
|
|
655
655
|
if (!type_entry) {
|
656
656
|
return LogicalType::INVALID;
|
657
657
|
}
|
658
|
-
|
659
|
-
EnumType::SetCatalog(result_type, type_entry.get());
|
660
|
-
return result_type;
|
658
|
+
return type_entry->user_type;
|
661
659
|
}
|
662
660
|
|
663
661
|
LogicalType Catalog::GetType(ClientContext &context, const string &catalog_name, const string &schema,
|
664
662
|
const string &name) {
|
665
663
|
auto &type_entry = Catalog::GetEntry<TypeCatalogEntry>(context, catalog_name, schema, name);
|
666
|
-
|
667
|
-
EnumType::SetCatalog(result_type, &type_entry);
|
668
|
-
return result_type;
|
664
|
+
return type_entry.user_type;
|
669
665
|
}
|
670
666
|
|
671
667
|
vector<reference<SchemaCatalogEntry>> Catalog::GetSchemas(ClientContext &context) {
|
@@ -527,10 +527,7 @@ unique_ptr<CatalogEntry> DuckTableEntry::DropNotNull(ClientContext &context, Dro
|
|
527
527
|
}
|
528
528
|
|
529
529
|
unique_ptr<CatalogEntry> DuckTableEntry::ChangeColumnType(ClientContext &context, ChangeColumnTypeInfo &info) {
|
530
|
-
|
531
|
-
info.target_type =
|
532
|
-
Catalog::GetType(context, catalog.GetName(), schema.name, UserType::GetTypeName(info.target_type));
|
533
|
-
}
|
530
|
+
Binder::BindLogicalType(context, info.target_type, &catalog, schema.name);
|
534
531
|
auto change_idx = GetColumnIndex(info.column_name);
|
535
532
|
auto create_info = make_uniq<CreateTableInfo>(schema, name);
|
536
533
|
create_info->temporary = temporary;
|
@@ -172,10 +172,6 @@ const ColumnList &TableCatalogEntry::GetColumns() const {
|
|
172
172
|
return columns;
|
173
173
|
}
|
174
174
|
|
175
|
-
ColumnList &TableCatalogEntry::GetColumnsMutable() {
|
176
|
-
return columns;
|
177
|
-
}
|
178
|
-
|
179
175
|
const ColumnDefinition &TableCatalogEntry::GetColumn(LogicalIndex idx) {
|
180
176
|
return columns.GetColumn(idx);
|
181
177
|
}
|
@@ -5,7 +5,6 @@
|
|
5
5
|
#include "duckdb/common/limits.hpp"
|
6
6
|
#include "duckdb/common/field_writer.hpp"
|
7
7
|
#include "duckdb/parser/keyword_helper.hpp"
|
8
|
-
#include "duckdb/parser/parsed_data/create_sequence_info.hpp"
|
9
8
|
#include "duckdb/common/types/vector.hpp"
|
10
9
|
#include <algorithm>
|
11
10
|
#include <sstream>
|
@@ -18,31 +17,13 @@ TypeCatalogEntry::TypeCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema,
|
|
18
17
|
this->internal = info.internal;
|
19
18
|
}
|
20
19
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
writer.AddField();
|
29
|
-
user_type.SerializeEnumType(writer.GetSerializer());
|
30
|
-
} else {
|
31
|
-
writer.WriteSerializable(user_type);
|
32
|
-
}
|
33
|
-
writer.Finalize();
|
34
|
-
}
|
35
|
-
|
36
|
-
unique_ptr<CreateTypeInfo> TypeCatalogEntry::Deserialize(Deserializer &source) {
|
37
|
-
auto info = make_uniq<CreateTypeInfo>();
|
38
|
-
|
39
|
-
FieldReader reader(source);
|
40
|
-
info->schema = reader.ReadRequired<string>();
|
41
|
-
info->name = reader.ReadRequired<string>();
|
42
|
-
info->type = reader.ReadRequiredSerializable<LogicalType, LogicalType>();
|
43
|
-
reader.Finalize();
|
44
|
-
|
45
|
-
return info;
|
20
|
+
unique_ptr<CreateInfo> TypeCatalogEntry::GetInfo() const {
|
21
|
+
auto result = make_uniq<CreateTypeInfo>();
|
22
|
+
result->catalog = catalog.GetName();
|
23
|
+
result->schema = schema.name;
|
24
|
+
result->name = name;
|
25
|
+
result->type = user_type;
|
26
|
+
return std::move(result);
|
46
27
|
}
|
47
28
|
|
48
29
|
string TypeCatalogEntry::ToSQL() const {
|
@@ -543,67 +543,6 @@ void CatalogSet::UpdateTimestamp(CatalogEntry &entry, transaction_t timestamp) {
|
|
543
543
|
mapping[entry.name]->timestamp = timestamp;
|
544
544
|
}
|
545
545
|
|
546
|
-
void CatalogSet::AdjustUserDependency(CatalogEntry &entry, ColumnDefinition &column, bool remove) {
|
547
|
-
auto user_type_catalog_p = EnumType::GetCatalog(column.Type());
|
548
|
-
if (!user_type_catalog_p) {
|
549
|
-
return;
|
550
|
-
}
|
551
|
-
auto &user_type_catalog = user_type_catalog_p->Cast<CatalogEntry>();
|
552
|
-
auto &dependency_manager = catalog.GetDependencyManager();
|
553
|
-
if (remove) {
|
554
|
-
dependency_manager.dependents_map[user_type_catalog].erase(*entry.parent);
|
555
|
-
dependency_manager.dependencies_map[*entry.parent].erase(user_type_catalog);
|
556
|
-
} else {
|
557
|
-
dependency_manager.dependents_map[user_type_catalog].insert(entry);
|
558
|
-
dependency_manager.dependencies_map[entry].insert(user_type_catalog);
|
559
|
-
}
|
560
|
-
}
|
561
|
-
|
562
|
-
void CatalogSet::AdjustDependency(CatalogEntry &entry, TableCatalogEntry &table, ColumnDefinition &column,
|
563
|
-
bool remove) {
|
564
|
-
bool found = false;
|
565
|
-
if (column.Type().id() == LogicalTypeId::ENUM) {
|
566
|
-
for (auto &old_column : table.GetColumns().Logical()) {
|
567
|
-
if (old_column.Name() == column.Name() && old_column.Type().id() != LogicalTypeId::ENUM) {
|
568
|
-
AdjustUserDependency(entry, column, remove);
|
569
|
-
found = true;
|
570
|
-
}
|
571
|
-
}
|
572
|
-
if (!found) {
|
573
|
-
AdjustUserDependency(entry, column, remove);
|
574
|
-
}
|
575
|
-
} else if (!(column.Type().GetAlias().empty())) {
|
576
|
-
auto alias = column.Type().GetAlias();
|
577
|
-
for (auto &old_column : table.GetColumns().Logical()) {
|
578
|
-
auto old_alias = old_column.Type().GetAlias();
|
579
|
-
if (old_column.Name() == column.Name() && old_alias != alias) {
|
580
|
-
AdjustUserDependency(entry, column, remove);
|
581
|
-
found = true;
|
582
|
-
}
|
583
|
-
}
|
584
|
-
if (!found) {
|
585
|
-
AdjustUserDependency(entry, column, remove);
|
586
|
-
}
|
587
|
-
}
|
588
|
-
}
|
589
|
-
|
590
|
-
void CatalogSet::AdjustTableDependencies(CatalogEntry &entry) {
|
591
|
-
if (entry.type == CatalogType::TABLE_ENTRY && entry.parent->type == CatalogType::TABLE_ENTRY) {
|
592
|
-
// If it's a table entry we have to check for possibly removing or adding user type dependencies
|
593
|
-
auto &old_table = entry.parent->Cast<TableCatalogEntry>();
|
594
|
-
auto &new_table = entry.Cast<TableCatalogEntry>();
|
595
|
-
|
596
|
-
for (idx_t i = 0; i < new_table.GetColumns().LogicalColumnCount(); i++) {
|
597
|
-
auto &new_column = new_table.GetColumnsMutable().GetColumnMutable(LogicalIndex(i));
|
598
|
-
AdjustDependency(entry, old_table, new_column, false);
|
599
|
-
}
|
600
|
-
for (idx_t i = 0; i < old_table.GetColumns().LogicalColumnCount(); i++) {
|
601
|
-
auto &old_column = old_table.GetColumnsMutable().GetColumnMutable(LogicalIndex(i));
|
602
|
-
AdjustDependency(entry, new_table, old_column, true);
|
603
|
-
}
|
604
|
-
}
|
605
|
-
}
|
606
|
-
|
607
546
|
void CatalogSet::Undo(CatalogEntry &entry) {
|
608
547
|
lock_guard<mutex> write_lock(catalog.GetWriteLock());
|
609
548
|
lock_guard<mutex> lock(catalog_lock);
|
@@ -614,8 +553,6 @@ void CatalogSet::Undo(CatalogEntry &entry) {
|
|
614
553
|
// i.e. we have to place (entry) as (entry->parent) again
|
615
554
|
auto &to_be_removed_node = *entry.parent;
|
616
555
|
|
617
|
-
AdjustTableDependencies(entry);
|
618
|
-
|
619
556
|
if (!to_be_removed_node.deleted) {
|
620
557
|
// delete the entry from the dependency manager as well
|
621
558
|
auto &dependency_manager = catalog.GetDependencyManager();
|
@@ -106,52 +106,16 @@ void DependencyManager::AlterObject(CatalogTransaction transaction, CatalogEntry
|
|
106
106
|
}
|
107
107
|
// add the new object to the dependents_map of each object that it depends on
|
108
108
|
auto &old_dependencies = dependencies_map[old_obj];
|
109
|
-
catalog_entry_vector_t to_delete;
|
110
109
|
for (auto &dep : old_dependencies) {
|
111
110
|
auto &dependency = dep.get();
|
112
|
-
if (dependency.type == CatalogType::TYPE_ENTRY) {
|
113
|
-
auto &user_type = dependency.Cast<TypeCatalogEntry>();
|
114
|
-
auto &table = new_obj.Cast<TableCatalogEntry>();
|
115
|
-
bool deleted_dependency = true;
|
116
|
-
for (auto &column : table.GetColumns().Logical()) {
|
117
|
-
if (column.Type() == user_type.user_type) {
|
118
|
-
deleted_dependency = false;
|
119
|
-
break;
|
120
|
-
}
|
121
|
-
}
|
122
|
-
if (deleted_dependency) {
|
123
|
-
to_delete.push_back(dependency);
|
124
|
-
continue;
|
125
|
-
}
|
126
|
-
}
|
127
111
|
dependents_map[dependency].insert(new_obj);
|
128
112
|
}
|
129
|
-
for (auto &dep : to_delete) {
|
130
|
-
auto &dependency = dep.get();
|
131
|
-
old_dependencies.erase(dependency);
|
132
|
-
dependents_map[dependency].erase(old_obj);
|
133
|
-
}
|
134
113
|
|
135
114
|
// We might have to add a type dependency
|
136
|
-
catalog_entry_vector_t to_add;
|
137
|
-
if (new_obj.type == CatalogType::TABLE_ENTRY) {
|
138
|
-
auto &table = new_obj.Cast<TableCatalogEntry>();
|
139
|
-
for (auto &column : table.GetColumns().Logical()) {
|
140
|
-
auto user_type_catalog = EnumType::GetCatalog(column.Type());
|
141
|
-
if (user_type_catalog) {
|
142
|
-
to_add.push_back(*user_type_catalog);
|
143
|
-
}
|
144
|
-
}
|
145
|
-
}
|
146
115
|
// add the new object to the dependency manager
|
147
116
|
dependents_map[new_obj] = dependency_set_t();
|
148
117
|
dependencies_map[new_obj] = old_dependencies;
|
149
118
|
|
150
|
-
for (auto &dependency : to_add) {
|
151
|
-
dependencies_map[new_obj].insert(dependency);
|
152
|
-
dependents_map[dependency].insert(new_obj);
|
153
|
-
}
|
154
|
-
|
155
119
|
for (auto &dependency : owned_objects_to_add) {
|
156
120
|
dependents_map[new_obj].insert(Dependency(dependency, DependencyType::DEPENDENCY_OWNS));
|
157
121
|
dependents_map[dependency].insert(Dependency(new_obj, DependencyType::DEPENDENCY_OWNED_BY));
|
@@ -299,8 +299,8 @@ PhysicalType EnumTypeInfo::DictType(idx_t size) {
|
|
299
299
|
|
300
300
|
template <class T>
|
301
301
|
struct EnumTypeInfoTemplated : public EnumTypeInfo {
|
302
|
-
explicit EnumTypeInfoTemplated(
|
303
|
-
: EnumTypeInfo(
|
302
|
+
explicit EnumTypeInfoTemplated(Vector &values_insert_order_p, idx_t size_p)
|
303
|
+
: EnumTypeInfo(values_insert_order_p, size_p) {
|
304
304
|
D_ASSERT(values_insert_order_p.GetType().InternalType() == PhysicalType::VARCHAR);
|
305
305
|
|
306
306
|
UnifiedVectorFormat vdata;
|
@@ -320,17 +320,16 @@ struct EnumTypeInfoTemplated : public EnumTypeInfo {
|
|
320
320
|
}
|
321
321
|
}
|
322
322
|
|
323
|
-
static shared_ptr<EnumTypeInfoTemplated> Deserialize(FieldReader &reader, uint32_t size
|
323
|
+
static shared_ptr<EnumTypeInfoTemplated> Deserialize(FieldReader &reader, uint32_t size) {
|
324
324
|
Vector values_insert_order(LogicalType::VARCHAR, size);
|
325
325
|
values_insert_order.Deserialize(size, reader.GetSource());
|
326
|
-
return make_shared<EnumTypeInfoTemplated>(
|
326
|
+
return make_shared<EnumTypeInfoTemplated>(values_insert_order, size);
|
327
327
|
}
|
328
328
|
|
329
329
|
static shared_ptr<EnumTypeInfoTemplated> FormatDeserialize(FormatDeserializer &source, uint32_t size) {
|
330
|
-
auto enum_name = source.ReadProperty<string>("enum_name");
|
331
330
|
Vector values_insert_order(LogicalType::VARCHAR, size);
|
332
331
|
values_insert_order.FormatDeserialize(source, size);
|
333
|
-
return make_shared<EnumTypeInfoTemplated>(
|
332
|
+
return make_shared<EnumTypeInfoTemplated>(values_insert_order, size);
|
334
333
|
}
|
335
334
|
|
336
335
|
const string_map_t<T> &GetValues() const {
|
@@ -344,23 +343,15 @@ private:
|
|
344
343
|
string_map_t<T> values;
|
345
344
|
};
|
346
345
|
|
347
|
-
EnumTypeInfo::EnumTypeInfo(
|
346
|
+
EnumTypeInfo::EnumTypeInfo(Vector &values_insert_order_p, idx_t dict_size_p)
|
348
347
|
: ExtraTypeInfo(ExtraTypeInfoType::ENUM_TYPE_INFO), values_insert_order(values_insert_order_p),
|
349
|
-
dict_type(EnumDictType::VECTOR_DICT),
|
348
|
+
dict_type(EnumDictType::VECTOR_DICT), dict_size(dict_size_p) {
|
350
349
|
}
|
351
350
|
|
352
351
|
const EnumDictType &EnumTypeInfo::GetEnumDictType() const {
|
353
352
|
return dict_type;
|
354
353
|
}
|
355
354
|
|
356
|
-
const string &EnumTypeInfo::GetEnumName() const {
|
357
|
-
return enum_name;
|
358
|
-
}
|
359
|
-
|
360
|
-
const string EnumTypeInfo::GetSchemaName() const {
|
361
|
-
return catalog_entry ? catalog_entry->schema.name : "";
|
362
|
-
}
|
363
|
-
|
364
355
|
const Vector &EnumTypeInfo::GetValuesInsertOrder() const {
|
365
356
|
return values_insert_order;
|
366
357
|
}
|
@@ -369,19 +360,19 @@ const idx_t &EnumTypeInfo::GetDictSize() const {
|
|
369
360
|
return dict_size;
|
370
361
|
}
|
371
362
|
|
372
|
-
LogicalType EnumTypeInfo::CreateType(
|
363
|
+
LogicalType EnumTypeInfo::CreateType(Vector &ordered_data, idx_t size) {
|
373
364
|
// Generate EnumTypeInfo
|
374
365
|
shared_ptr<ExtraTypeInfo> info;
|
375
366
|
auto enum_internal_type = EnumTypeInfo::DictType(size);
|
376
367
|
switch (enum_internal_type) {
|
377
368
|
case PhysicalType::UINT8:
|
378
|
-
info = make_shared<EnumTypeInfoTemplated<uint8_t>>(
|
369
|
+
info = make_shared<EnumTypeInfoTemplated<uint8_t>>(ordered_data, size);
|
379
370
|
break;
|
380
371
|
case PhysicalType::UINT16:
|
381
|
-
info = make_shared<EnumTypeInfoTemplated<uint16_t>>(
|
372
|
+
info = make_shared<EnumTypeInfoTemplated<uint16_t>>(ordered_data, size);
|
382
373
|
break;
|
383
374
|
case PhysicalType::UINT32:
|
384
|
-
info = make_shared<EnumTypeInfoTemplated<uint32_t>>(
|
375
|
+
info = make_shared<EnumTypeInfoTemplated<uint32_t>>(ordered_data, size);
|
385
376
|
break;
|
386
377
|
default:
|
387
378
|
throw InternalException("Invalid Physical Type for ENUMs");
|
@@ -413,37 +404,22 @@ int64_t EnumType::GetPos(const LogicalType &type, const string_t &key) {
|
|
413
404
|
}
|
414
405
|
}
|
415
406
|
|
407
|
+
string_t EnumType::GetString(const LogicalType &type, idx_t pos) {
|
408
|
+
D_ASSERT(pos < EnumType::GetSize(type));
|
409
|
+
return FlatVector::GetData<string_t>(EnumType::GetValuesInsertOrder(type))[pos];
|
410
|
+
}
|
411
|
+
|
416
412
|
shared_ptr<ExtraTypeInfo> EnumTypeInfo::Deserialize(FieldReader &reader) {
|
417
|
-
auto schema_name = reader.ReadRequired<string>();
|
418
|
-
auto enum_name = reader.ReadRequired<string>();
|
419
|
-
auto deserialize_internals = reader.ReadRequired<bool>();
|
420
|
-
if (!deserialize_internals) {
|
421
|
-
// this means the enum should already be in the catalog.
|
422
|
-
auto &client_context = reader.GetSource().GetContext();
|
423
|
-
// See if the serializer has a catalog
|
424
|
-
auto catalog = reader.GetSource().GetCatalog();
|
425
|
-
shared_ptr<ExtraTypeInfo> extra_info;
|
426
|
-
if (catalog) {
|
427
|
-
auto enum_type = catalog->GetType(client_context, schema_name, enum_name, OnEntryNotFound::RETURN_NULL);
|
428
|
-
if (enum_type != LogicalType::INVALID) {
|
429
|
-
extra_info = enum_type.GetAuxInfoShrPtr();
|
430
|
-
}
|
431
|
-
}
|
432
|
-
if (!extra_info) {
|
433
|
-
throw InternalException("Could not find ENUM in the Catalog to deserialize");
|
434
|
-
}
|
435
|
-
return extra_info;
|
436
|
-
}
|
437
413
|
// deserialize the enum data
|
438
414
|
auto enum_size = reader.ReadRequired<uint32_t>();
|
439
415
|
auto enum_internal_type = EnumTypeInfo::DictType(enum_size);
|
440
416
|
switch (enum_internal_type) {
|
441
417
|
case PhysicalType::UINT8:
|
442
|
-
return EnumTypeInfoTemplated<uint8_t>::Deserialize(reader, enum_size
|
418
|
+
return EnumTypeInfoTemplated<uint8_t>::Deserialize(reader, enum_size);
|
443
419
|
case PhysicalType::UINT16:
|
444
|
-
return EnumTypeInfoTemplated<uint16_t>::Deserialize(reader, enum_size
|
420
|
+
return EnumTypeInfoTemplated<uint16_t>::Deserialize(reader, enum_size);
|
445
421
|
case PhysicalType::UINT32:
|
446
|
-
return EnumTypeInfoTemplated<uint32_t>::Deserialize(reader, enum_size
|
422
|
+
return EnumTypeInfoTemplated<uint32_t>::Deserialize(reader, enum_size);
|
447
423
|
default:
|
448
424
|
throw InternalException("Invalid Physical Type for ENUMs");
|
449
425
|
}
|
@@ -491,14 +467,16 @@ void EnumTypeInfo::Serialize(FieldWriter &writer) const {
|
|
491
467
|
if (dict_type != EnumDictType::VECTOR_DICT) {
|
492
468
|
throw InternalException("Cannot serialize non-vector dictionary ENUM types");
|
493
469
|
}
|
494
|
-
|
495
|
-
|
470
|
+
auto dict_size = GetDictSize();
|
471
|
+
// Store Dictionary Size
|
472
|
+
writer.WriteField<uint32_t>(dict_size);
|
473
|
+
// Store Vector Order By Insertion
|
474
|
+
((Vector &)GetValuesInsertOrder()).Serialize(dict_size, writer.GetSerializer()); // NOLINT - FIXME
|
496
475
|
}
|
497
476
|
|
498
477
|
void EnumTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
|
499
478
|
ExtraTypeInfo::FormatSerialize(serializer);
|
500
479
|
serializer.WriteProperty("dict_size", dict_size);
|
501
|
-
serializer.WriteProperty("enum_name", enum_name);
|
502
480
|
((Vector &)values_insert_order).FormatSerialize(serializer, dict_size); // NOLINT - FIXME
|
503
481
|
}
|
504
482
|
|
@@ -8,7 +8,6 @@ namespace duckdb {
|
|
8
8
|
FieldWriter::FieldWriter(Serializer &serializer_p)
|
9
9
|
: serializer(serializer_p), buffer(make_uniq<BufferedSerializer>()), field_count(0), finalized(false) {
|
10
10
|
buffer->SetVersion(serializer.GetVersion());
|
11
|
-
buffer->is_query_plan = serializer.is_query_plan;
|
12
11
|
}
|
13
12
|
|
14
13
|
FieldWriter::~FieldWriter() {
|
@@ -71,11 +71,11 @@ string FileSystem::GetEnvVariable(const string &name) {
|
|
71
71
|
}
|
72
72
|
|
73
73
|
bool FileSystem::IsPathAbsolute(const string &path) {
|
74
|
-
auto path_separator =
|
74
|
+
auto path_separator = PathSeparator(path);
|
75
75
|
return PathMatched(path, path_separator);
|
76
76
|
}
|
77
77
|
|
78
|
-
string FileSystem::PathSeparator() {
|
78
|
+
string FileSystem::PathSeparator(const string &path) {
|
79
79
|
return "/";
|
80
80
|
}
|
81
81
|
|
@@ -167,7 +167,7 @@ string FileSystem::NormalizeAbsolutePath(const string &path) {
|
|
167
167
|
return result;
|
168
168
|
}
|
169
169
|
|
170
|
-
string FileSystem::PathSeparator() {
|
170
|
+
string FileSystem::PathSeparator(const string &path) {
|
171
171
|
return "\\";
|
172
172
|
}
|
173
173
|
|
@@ -210,11 +210,11 @@ string FileSystem::GetWorkingDirectory() {
|
|
210
210
|
|
211
211
|
string FileSystem::JoinPath(const string &a, const string &b) {
|
212
212
|
// FIXME: sanitize paths
|
213
|
-
return a + PathSeparator() + b;
|
213
|
+
return a + PathSeparator(a) + b;
|
214
214
|
}
|
215
215
|
|
216
216
|
string FileSystem::ConvertSeparators(const string &path) {
|
217
|
-
auto separator_str = PathSeparator();
|
217
|
+
auto separator_str = PathSeparator(path);
|
218
218
|
char separator = separator_str[0];
|
219
219
|
if (separator == '/') {
|
220
220
|
// on unix-based systems we only accept / as a separator
|
@@ -229,7 +229,7 @@ string FileSystem::ExtractName(const string &path) {
|
|
229
229
|
return string();
|
230
230
|
}
|
231
231
|
auto normalized_path = ConvertSeparators(path);
|
232
|
-
auto sep = PathSeparator();
|
232
|
+
auto sep = PathSeparator(path);
|
233
233
|
auto splits = StringUtil::Split(normalized_path, sep);
|
234
234
|
D_ASSERT(!splits.empty());
|
235
235
|
return splits.back();
|
@@ -24,7 +24,7 @@ void FilenamePattern::SetFilenamePattern(const string &pattern) {
|
|
24
24
|
_pos = std::min(_pos, (idx_t)_base.length());
|
25
25
|
}
|
26
26
|
|
27
|
-
string FilenamePattern::CreateFilename(
|
27
|
+
string FilenamePattern::CreateFilename(FileSystem &fs, const string &path, const string &extension,
|
28
28
|
idx_t offset) const {
|
29
29
|
string result(_base);
|
30
30
|
string replacement;
|
@@ -145,6 +145,13 @@ void MiniZStreamWrapper::Initialize(CompressedFile &file, bool write) {
|
|
145
145
|
bool MiniZStreamWrapper::Read(StreamData &sd) {
|
146
146
|
// Handling for the concatenated files
|
147
147
|
if (sd.refresh) {
|
148
|
+
auto available = (uint32_t)(sd.in_buff_end - sd.in_buff_start);
|
149
|
+
if (available <= GZIP_FOOTER_SIZE) {
|
150
|
+
// Only footer is available so we just close and return finished
|
151
|
+
Close();
|
152
|
+
return true;
|
153
|
+
}
|
154
|
+
|
148
155
|
sd.refresh = false;
|
149
156
|
auto body_ptr = sd.in_buff_start + GZIP_FOOTER_SIZE;
|
150
157
|
uint8_t gzip_hdr[GZIP_HEADER_MINSIZE];
|
@@ -200,18 +207,6 @@ bool MiniZStreamWrapper::Read(StreamData &sd) {
|
|
200
207
|
|
201
208
|
// if stream ended, deallocate inflator
|
202
209
|
if (ret == duckdb_miniz::MZ_STREAM_END) {
|
203
|
-
// Last read from file done and remaining bytes only for footer or less
|
204
|
-
if ((sd.in_buff_end < sd.in_buff.get() + sd.in_buf_size) && mz_stream_ptr->avail_in <= GZIP_FOOTER_SIZE) {
|
205
|
-
Close();
|
206
|
-
return true;
|
207
|
-
}
|
208
|
-
if (mz_stream_ptr->avail_in > GZIP_FOOTER_SIZE) {
|
209
|
-
// Definitely not concatenated gzip
|
210
|
-
if (*(sd.in_buff_start + GZIP_FOOTER_SIZE) != 0x1F) {
|
211
|
-
Close();
|
212
|
-
return true;
|
213
|
-
}
|
214
|
-
}
|
215
210
|
// Concatenated GZIP potentially coming up - refresh input buffer
|
216
211
|
sd.refresh = true;
|
217
212
|
}
|
@@ -440,10 +440,11 @@ void UnionByName::CombineUnionTypes(const vector<string> &col_names, const vecto
|
|
440
440
|
}
|
441
441
|
}
|
442
442
|
|
443
|
-
bool MultiFileReaderOptions::AutoDetectHivePartitioningInternal(const vector<string> &files) {
|
443
|
+
bool MultiFileReaderOptions::AutoDetectHivePartitioningInternal(const vector<string> &files, ClientContext &context) {
|
444
444
|
std::unordered_set<string> partitions;
|
445
|
+
auto &fs = FileSystem::GetFileSystem(context);
|
445
446
|
|
446
|
-
auto splits_first_file = StringUtil::Split(files.front(),
|
447
|
+
auto splits_first_file = StringUtil::Split(files.front(), fs.PathSeparator(files.front()));
|
447
448
|
if (splits_first_file.size() < 2) {
|
448
449
|
return false;
|
449
450
|
}
|
@@ -457,7 +458,7 @@ bool MultiFileReaderOptions::AutoDetectHivePartitioningInternal(const vector<str
|
|
457
458
|
return false;
|
458
459
|
}
|
459
460
|
for (auto &file : files) {
|
460
|
-
auto splits = StringUtil::Split(file,
|
461
|
+
auto splits = StringUtil::Split(file, fs.PathSeparator(file));
|
461
462
|
if (splits.size() != splits_first_file.size()) {
|
462
463
|
return false;
|
463
464
|
}
|
@@ -474,8 +475,10 @@ bool MultiFileReaderOptions::AutoDetectHivePartitioningInternal(const vector<str
|
|
474
475
|
return true;
|
475
476
|
}
|
476
477
|
void MultiFileReaderOptions::AutoDetectHiveTypesInternal(const string &file, ClientContext &context) {
|
478
|
+
auto &fs = FileSystem::GetFileSystem(context);
|
479
|
+
|
477
480
|
std::map<string, string> partitions;
|
478
|
-
auto splits = StringUtil::Split(file,
|
481
|
+
auto splits = StringUtil::Split(file, fs.PathSeparator(file));
|
479
482
|
if (splits.size() < 2) {
|
480
483
|
return;
|
481
484
|
}
|
@@ -518,7 +521,7 @@ void MultiFileReaderOptions::AutoDetectHivePartitioning(const vector<string> &fi
|
|
518
521
|
auto_detect_hive_partitioning = false;
|
519
522
|
}
|
520
523
|
if (auto_detect_hive_partitioning) {
|
521
|
-
hive_partitioning = AutoDetectHivePartitioningInternal(files);
|
524
|
+
hive_partitioning = AutoDetectHivePartitioningInternal(files, context);
|
522
525
|
}
|
523
526
|
if (hive_partitioning && hive_types_autocast) {
|
524
527
|
AutoDetectHiveTypesInternal(files.front(), context);
|
@@ -63,13 +63,4 @@ ClientContext &BufferedFileReader::GetContext() {
|
|
63
63
|
return *context;
|
64
64
|
}
|
65
65
|
|
66
|
-
optional_ptr<Catalog> BufferedFileReader::GetCatalog() {
|
67
|
-
return catalog;
|
68
|
-
}
|
69
|
-
|
70
|
-
void BufferedFileReader::SetCatalog(Catalog &catalog_p) {
|
71
|
-
D_ASSERT(!catalog);
|
72
|
-
this->catalog = &catalog_p;
|
73
|
-
}
|
74
|
-
|
75
66
|
} // namespace duckdb
|
@@ -395,7 +395,15 @@ string LogicalType::ToString() const {
|
|
395
395
|
return StringUtil::Format("DECIMAL(%d,%d)", width, scale);
|
396
396
|
}
|
397
397
|
case LogicalTypeId::ENUM: {
|
398
|
-
|
398
|
+
string ret = "ENUM(";
|
399
|
+
for (idx_t i = 0; i < EnumType::GetSize(*this); i++) {
|
400
|
+
if (i > 0) {
|
401
|
+
ret += ", ";
|
402
|
+
}
|
403
|
+
ret += "'" + KeywordHelper::WriteOptionallyQuoted(EnumType::GetString(*this, i).GetString(), '\'') + "'";
|
404
|
+
}
|
405
|
+
ret += ")";
|
406
|
+
return ret;
|
399
407
|
}
|
400
408
|
case LogicalTypeId::USER: {
|
401
409
|
return KeywordHelper::WriteOptionallyQuoted(UserType::GetTypeName(*this));
|
@@ -995,34 +1003,12 @@ LogicalType LogicalType::USER(const string &user_type_name) {
|
|
995
1003
|
//===--------------------------------------------------------------------===//
|
996
1004
|
// Enum Type
|
997
1005
|
//===--------------------------------------------------------------------===//
|
998
|
-
|
999
|
-
|
1000
|
-
auto &enum_info = type_info.Cast<EnumTypeInfo>();
|
1001
|
-
// Store Schema Name
|
1002
|
-
writer.WriteString(enum_info.GetSchemaName());
|
1003
|
-
// Store Enum Name
|
1004
|
-
writer.WriteString(enum_info.GetEnumName());
|
1005
|
-
// Store If we are serializing the internals
|
1006
|
-
writer.WriteField<bool>(serialize_internals);
|
1007
|
-
if (serialize_internals) {
|
1008
|
-
// We must serialize the internals
|
1009
|
-
auto dict_size = enum_info.GetDictSize();
|
1010
|
-
// Store Dictionary Size
|
1011
|
-
writer.WriteField<uint32_t>(dict_size);
|
1012
|
-
// Store Vector Order By Insertion
|
1013
|
-
((Vector &)enum_info.GetValuesInsertOrder()).Serialize(dict_size, writer.GetSerializer()); // NOLINT - FIXME
|
1014
|
-
}
|
1015
|
-
}
|
1016
|
-
|
1017
|
-
const string &EnumType::GetTypeName(const LogicalType &type) {
|
1018
|
-
D_ASSERT(type.id() == LogicalTypeId::ENUM);
|
1019
|
-
auto info = type.AuxInfo();
|
1020
|
-
D_ASSERT(info);
|
1021
|
-
return info->Cast<EnumTypeInfo>().GetEnumName();
|
1006
|
+
LogicalType LogicalType::ENUM(Vector &ordered_data, idx_t size) {
|
1007
|
+
return EnumTypeInfo::CreateType(ordered_data, size);
|
1022
1008
|
}
|
1023
1009
|
|
1024
1010
|
LogicalType LogicalType::ENUM(const string &enum_name, Vector &ordered_data, idx_t size) {
|
1025
|
-
return
|
1011
|
+
return LogicalType::ENUM(ordered_data, size);
|
1026
1012
|
}
|
1027
1013
|
|
1028
1014
|
const string EnumType::GetValue(const Value &val) {
|
@@ -1045,27 +1031,6 @@ idx_t EnumType::GetSize(const LogicalType &type) {
|
|
1045
1031
|
return info->Cast<EnumTypeInfo>().GetDictSize();
|
1046
1032
|
}
|
1047
1033
|
|
1048
|
-
void EnumType::SetCatalog(LogicalType &type, optional_ptr<TypeCatalogEntry> catalog_entry) {
|
1049
|
-
auto info = type.AuxInfo();
|
1050
|
-
if (!info) {
|
1051
|
-
return;
|
1052
|
-
}
|
1053
|
-
((ExtraTypeInfo &)*info).catalog_entry = catalog_entry;
|
1054
|
-
}
|
1055
|
-
|
1056
|
-
optional_ptr<TypeCatalogEntry> EnumType::GetCatalog(const LogicalType &type) {
|
1057
|
-
auto info = type.AuxInfo();
|
1058
|
-
if (!info) {
|
1059
|
-
return nullptr;
|
1060
|
-
}
|
1061
|
-
return info->catalog_entry;
|
1062
|
-
}
|
1063
|
-
|
1064
|
-
string EnumType::GetSchemaName(const LogicalType &type) {
|
1065
|
-
auto catalog_entry = EnumType::GetCatalog(type);
|
1066
|
-
return catalog_entry ? catalog_entry->schema.name : "";
|
1067
|
-
}
|
1068
|
-
|
1069
1034
|
PhysicalType EnumType::GetPhysicalType(const LogicalType &type) {
|
1070
1035
|
D_ASSERT(type.id() == LogicalTypeId::ENUM);
|
1071
1036
|
auto aux_info = type.AuxInfo();
|
@@ -1090,15 +1055,6 @@ void LogicalType::Serialize(Serializer &serializer) const {
|
|
1090
1055
|
writer.Finalize();
|
1091
1056
|
}
|
1092
1057
|
|
1093
|
-
void LogicalType::SerializeEnumType(Serializer &serializer) const {
|
1094
|
-
FieldWriter writer(serializer);
|
1095
|
-
writer.WriteField<LogicalTypeId>(id_);
|
1096
|
-
writer.WriteField<ExtraTypeInfoType>(type_info_->type);
|
1097
|
-
EnumType::Serialize(writer, *type_info_, true);
|
1098
|
-
writer.WriteString(type_info_->alias);
|
1099
|
-
writer.Finalize();
|
1100
|
-
}
|
1101
|
-
|
1102
1058
|
LogicalType LogicalType::Deserialize(Deserializer &source) {
|
1103
1059
|
FieldReader reader(source);
|
1104
1060
|
auto id = reader.ReadRequired<LogicalTypeId>();
|
@@ -108,6 +108,10 @@ void VirtualFileSystem::RemoveFile(const string &filename) {
|
|
108
108
|
FindFileSystem(filename).RemoveFile(filename);
|
109
109
|
}
|
110
110
|
|
111
|
+
string VirtualFileSystem::PathSeparator(const string &path) {
|
112
|
+
return FindFileSystem(path).PathSeparator(path);
|
113
|
+
}
|
114
|
+
|
111
115
|
vector<string> VirtualFileSystem::Glob(const string &path, FileOpener *opener) {
|
112
116
|
return FindFileSystem(path).Glob(path, opener);
|
113
117
|
}
|