duckdb 0.7.1-dev90.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/README.md +1 -1
  2. package/binding.gyp +7 -7
  3. package/package.json +3 -3
  4. package/src/duckdb/extension/json/buffered_json_reader.cpp +50 -9
  5. package/src/duckdb/extension/json/include/buffered_json_reader.hpp +7 -2
  6. package/src/duckdb/extension/json/include/json_scan.hpp +45 -10
  7. package/src/duckdb/extension/json/json_functions/copy_json.cpp +35 -22
  8. package/src/duckdb/extension/json/json_functions/json_create.cpp +8 -8
  9. package/src/duckdb/extension/json/json_functions/json_structure.cpp +8 -3
  10. package/src/duckdb/extension/json/json_functions/json_transform.cpp +54 -10
  11. package/src/duckdb/extension/json/json_functions/read_json.cpp +104 -49
  12. package/src/duckdb/extension/json/json_functions/read_json_objects.cpp +5 -3
  13. package/src/duckdb/extension/json/json_functions.cpp +7 -0
  14. package/src/duckdb/extension/json/json_scan.cpp +144 -38
  15. package/src/duckdb/extension/parquet/column_reader.cpp +7 -0
  16. package/src/duckdb/extension/parquet/include/column_reader.hpp +1 -0
  17. package/src/duckdb/extension/parquet/parquet-extension.cpp +2 -10
  18. package/src/duckdb/src/catalog/catalog.cpp +62 -13
  19. package/src/duckdb/src/catalog/catalog_entry/index_catalog_entry.cpp +8 -7
  20. package/src/duckdb/src/catalog/catalog_entry/schema_catalog_entry.cpp +1 -1
  21. package/src/duckdb/src/catalog/catalog_set.cpp +1 -1
  22. package/src/duckdb/src/catalog/default/default_functions.cpp +1 -0
  23. package/src/duckdb/src/catalog/default/default_views.cpp +1 -1
  24. package/src/duckdb/src/common/bind_helpers.cpp +55 -0
  25. package/src/duckdb/src/common/file_system.cpp +23 -9
  26. package/src/duckdb/src/common/hive_partitioning.cpp +1 -0
  27. package/src/duckdb/src/common/local_file_system.cpp +4 -4
  28. package/src/duckdb/src/common/string_util.cpp +8 -4
  29. package/src/duckdb/src/common/types/partitioned_column_data.cpp +1 -0
  30. package/src/duckdb/src/common/types.cpp +37 -11
  31. package/src/duckdb/src/execution/column_binding_resolver.cpp +5 -2
  32. package/src/duckdb/src/execution/index/art/art.cpp +117 -67
  33. package/src/duckdb/src/execution/index/art/art_key.cpp +24 -12
  34. package/src/duckdb/src/execution/index/art/leaf.cpp +7 -8
  35. package/src/duckdb/src/execution/index/art/node.cpp +13 -27
  36. package/src/duckdb/src/execution/index/art/node16.cpp +5 -8
  37. package/src/duckdb/src/execution/index/art/node256.cpp +3 -5
  38. package/src/duckdb/src/execution/index/art/node4.cpp +4 -7
  39. package/src/duckdb/src/execution/index/art/node48.cpp +5 -8
  40. package/src/duckdb/src/execution/index/art/prefix.cpp +2 -3
  41. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +6 -27
  42. package/src/duckdb/src/execution/operator/helper/physical_reset.cpp +1 -9
  43. package/src/duckdb/src/execution/operator/helper/physical_set.cpp +1 -9
  44. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +7 -9
  45. package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp +9 -0
  46. package/src/duckdb/src/execution/physical_operator.cpp +6 -6
  47. package/src/duckdb/src/function/pragma/pragma_queries.cpp +38 -11
  48. package/src/duckdb/src/function/scalar/generic/current_setting.cpp +2 -2
  49. package/src/duckdb/src/function/scalar/list/array_slice.cpp +2 -3
  50. package/src/duckdb/src/function/scalar/map/map.cpp +69 -21
  51. package/src/duckdb/src/function/scalar/string/like.cpp +6 -3
  52. package/src/duckdb/src/function/table/read_csv.cpp +16 -5
  53. package/src/duckdb/src/function/table/system/duckdb_temporary_files.cpp +59 -0
  54. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  55. package/src/duckdb/src/function/table/table_scan.cpp +3 -0
  56. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  57. package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +7 -1
  58. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_index_entry.hpp +1 -1
  59. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/index_catalog_entry.hpp +1 -1
  60. package/src/duckdb/src/include/duckdb/common/bind_helpers.hpp +2 -0
  61. package/src/duckdb/src/include/duckdb/common/enums/statement_type.hpp +1 -1
  62. package/src/duckdb/src/include/duckdb/common/enums/wal_type.hpp +3 -0
  63. package/src/duckdb/src/include/duckdb/common/file_system.hpp +1 -1
  64. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +9 -1
  65. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +4 -4
  66. package/src/duckdb/src/include/duckdb/common/string_util.hpp +9 -2
  67. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +37 -41
  68. package/src/duckdb/src/include/duckdb/execution/index/art/art_key.hpp +8 -11
  69. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +2 -0
  70. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -1
  71. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
  72. package/src/duckdb/src/include/duckdb/main/client_data.hpp +2 -2
  73. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  74. package/src/duckdb/src/include/duckdb/main/{extension_functions.hpp → extension_entries.hpp} +27 -5
  75. package/src/duckdb/src/include/duckdb/main/extension_helper.hpp +11 -1
  76. package/src/duckdb/src/include/duckdb/main/settings.hpp +9 -0
  77. package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp +0 -7
  78. package/src/duckdb/src/include/duckdb/parser/query_node/select_node.hpp +1 -1
  79. package/src/duckdb/src/include/duckdb/parser/sql_statement.hpp +2 -2
  80. package/src/duckdb/src/include/duckdb/parser/statement/copy_statement.hpp +1 -1
  81. package/src/duckdb/src/include/duckdb/parser/statement/select_statement.hpp +3 -3
  82. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +1 -1
  83. package/src/duckdb/src/include/duckdb/planner/binder.hpp +3 -0
  84. package/src/duckdb/src/include/duckdb/planner/expression_binder/index_binder.hpp +10 -3
  85. package/src/duckdb/src/include/duckdb/planner/operator/logical_execute.hpp +1 -5
  86. package/src/duckdb/src/include/duckdb/planner/operator/logical_show.hpp +1 -2
  87. package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp +8 -0
  88. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +7 -1
  89. package/src/duckdb/src/include/duckdb/storage/index.hpp +47 -38
  90. package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp +7 -0
  91. package/src/duckdb/src/main/client_context.cpp +2 -0
  92. package/src/duckdb/src/main/config.cpp +1 -0
  93. package/src/duckdb/src/main/database.cpp +14 -5
  94. package/src/duckdb/src/main/extension/extension_alias.cpp +2 -1
  95. package/src/duckdb/src/main/extension/extension_helper.cpp +15 -0
  96. package/src/duckdb/src/main/extension/extension_install.cpp +60 -16
  97. package/src/duckdb/src/main/extension/extension_load.cpp +62 -13
  98. package/src/duckdb/src/main/settings/settings.cpp +16 -0
  99. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +2 -6
  100. package/src/duckdb/src/parallel/pipeline_executor.cpp +1 -55
  101. package/src/duckdb/src/parser/parsed_data/create_index_info.cpp +3 -0
  102. package/src/duckdb/src/parser/statement/copy_statement.cpp +2 -13
  103. package/src/duckdb/src/parser/statement/delete_statement.cpp +3 -0
  104. package/src/duckdb/src/parser/statement/insert_statement.cpp +9 -0
  105. package/src/duckdb/src/parser/statement/update_statement.cpp +3 -0
  106. package/src/duckdb/src/parser/transform/expression/transform_case.cpp +3 -3
  107. package/src/duckdb/src/planner/bind_context.cpp +1 -1
  108. package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp +3 -0
  109. package/src/duckdb/src/planner/binder/statement/bind_copy.cpp +7 -14
  110. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +13 -0
  111. package/src/duckdb/src/planner/binder/statement/bind_drop.cpp +2 -2
  112. package/src/duckdb/src/planner/binder/statement/bind_insert.cpp +22 -1
  113. package/src/duckdb/src/planner/expression_binder/index_binder.cpp +32 -1
  114. package/src/duckdb/src/planner/logical_operator.cpp +4 -1
  115. package/src/duckdb/src/storage/buffer_manager.cpp +105 -26
  116. package/src/duckdb/src/storage/compression/bitpacking.cpp +16 -7
  117. package/src/duckdb/src/storage/data_table.cpp +66 -3
  118. package/src/duckdb/src/storage/index.cpp +1 -1
  119. package/src/duckdb/src/storage/local_storage.cpp +1 -1
  120. package/src/duckdb/src/storage/table_index_list.cpp +1 -2
  121. package/src/duckdb/src/storage/wal_replay.cpp +68 -0
  122. package/src/duckdb/src/storage/write_ahead_log.cpp +21 -1
  123. package/src/duckdb/src/transaction/commit_state.cpp +5 -2
  124. package/src/duckdb/third_party/concurrentqueue/blockingconcurrentqueue.h +2 -2
  125. package/src/duckdb/third_party/fmt/include/fmt/core.h +1 -2
  126. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +4 -4
  127. package/src/duckdb/ub_src_function_table_system.cpp +2 -0
  128. package/src/statement.cpp +46 -12
  129. package/test/arrow.test.ts +3 -3
  130. package/test/prepare.test.ts +39 -1
  131. package/test/typescript_decls.test.ts +1 -1
@@ -115,6 +115,21 @@ DefaultExtension ExtensionHelper::GetDefaultExtension(idx_t index) {
115
115
  return internal_extensions[index];
116
116
  }
117
117
 
118
+ //===--------------------------------------------------------------------===//
119
+ // Allow Auto-Install Extensions
120
+ //===--------------------------------------------------------------------===//
121
+ static const char *auto_install[] = {"motherduck", "postgres_scanner", "sqlite_scanner", nullptr};
122
+
123
+ bool ExtensionHelper::AllowAutoInstall(const string &extension) {
124
+ auto lcase = StringUtil::Lower(extension);
125
+ for (idx_t i = 0; auto_install[i]; i++) {
126
+ if (lcase == auto_install[i]) {
127
+ return true;
128
+ }
129
+ }
130
+ return false;
131
+ }
132
+
118
133
  //===--------------------------------------------------------------------===//
119
134
  // Load Statically Compiled Extension
120
135
  //===--------------------------------------------------------------------===//
@@ -38,22 +38,57 @@ const vector<string> ExtensionHelper::PathComponents() {
38
38
  return vector<string> {".duckdb", "extensions", GetVersionDirectoryName(), DuckDB::Platform()};
39
39
  }
40
40
 
41
- string ExtensionHelper::ExtensionDirectory(ClientContext &context) {
42
- auto &fs = FileSystem::GetFileSystem(context);
43
- string local_path = fs.GetHomeDirectory(FileSystem::GetFileOpener(context));
44
- if (!fs.DirectoryExists(local_path)) {
45
- throw IOException("Can't find the home directory at '%s'\nSpecify a home directory using the SET "
46
- "home_directory='/path/to/dir' option.",
47
- local_path);
41
+ string ExtensionHelper::ExtensionDirectory(DBConfig &config, FileSystem &fs, FileOpener *opener) {
42
+ string extension_directory;
43
+ if (!config.options.extension_directory.empty()) { // create the extension directory if not present
44
+ extension_directory = config.options.extension_directory;
45
+ // TODO this should probably live in the FileSystem
46
+ // convert random separators to platform-canonic
47
+ extension_directory = fs.ConvertSeparators(extension_directory);
48
+ // expand ~ in extension directory
49
+ extension_directory = fs.ExpandPath(extension_directory, opener);
50
+ if (!fs.DirectoryExists(extension_directory)) {
51
+ auto sep = fs.PathSeparator();
52
+ auto splits = StringUtil::Split(extension_directory, sep);
53
+ D_ASSERT(!splits.empty());
54
+ string extension_directory_prefix;
55
+ if (StringUtil::StartsWith(extension_directory, sep)) {
56
+ extension_directory_prefix = sep; // this is swallowed by Split otherwise
57
+ }
58
+ for (auto &split : splits) {
59
+ extension_directory_prefix = extension_directory_prefix + split + sep;
60
+ if (!fs.DirectoryExists(extension_directory_prefix)) {
61
+ fs.CreateDirectory(extension_directory_prefix);
62
+ }
63
+ }
64
+ }
65
+ } else { // otherwise default to home
66
+ string home_directory = fs.GetHomeDirectory(opener);
67
+ // exception if the home directory does not exist, don't create whatever we think is home
68
+ if (!fs.DirectoryExists(home_directory)) {
69
+ throw IOException("Can't find the home directory at '%s'\nSpecify a home directory using the SET "
70
+ "home_directory='/path/to/dir' option.",
71
+ home_directory);
72
+ }
73
+ extension_directory = home_directory;
48
74
  }
75
+ D_ASSERT(fs.DirectoryExists(extension_directory));
76
+
49
77
  auto path_components = PathComponents();
50
78
  for (auto &path_ele : path_components) {
51
- local_path = fs.JoinPath(local_path, path_ele);
52
- if (!fs.DirectoryExists(local_path)) {
53
- fs.CreateDirectory(local_path);
79
+ extension_directory = fs.JoinPath(extension_directory, path_ele);
80
+ if (!fs.DirectoryExists(extension_directory)) {
81
+ fs.CreateDirectory(extension_directory);
54
82
  }
55
83
  }
56
- return local_path;
84
+ return extension_directory;
85
+ }
86
+
87
+ string ExtensionHelper::ExtensionDirectory(ClientContext &context) {
88
+ auto &config = DBConfig::GetConfig(context);
89
+ auto &fs = FileSystem::GetFileSystem(context);
90
+ auto opener = FileSystem::GetFileOpener(context);
91
+ return ExtensionDirectory(config, fs, opener);
57
92
  }
58
93
 
59
94
  bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &message) {
@@ -75,15 +110,24 @@ bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &me
75
110
  return false;
76
111
  }
77
112
 
113
+ void ExtensionHelper::InstallExtension(DBConfig &config, FileSystem &fs, const string &extension, bool force_install) {
114
+ string local_path = ExtensionDirectory(config, fs, nullptr);
115
+ InstallExtensionInternal(config, nullptr, fs, local_path, extension, force_install);
116
+ }
117
+
78
118
  void ExtensionHelper::InstallExtension(ClientContext &context, const string &extension, bool force_install) {
79
119
  auto &config = DBConfig::GetConfig(context);
80
- if (!config.options.enable_external_access) {
81
- throw PermissionException("Installing extensions is disabled through configuration");
82
- }
83
120
  auto &fs = FileSystem::GetFileSystem(context);
84
-
85
121
  string local_path = ExtensionDirectory(context);
122
+ auto &client_config = ClientConfig::GetConfig(context);
123
+ InstallExtensionInternal(config, &client_config, fs, local_path, extension, force_install);
124
+ }
86
125
 
126
+ void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *client_config, FileSystem &fs,
127
+ const string &local_path, const string &extension, bool force_install) {
128
+ if (!config.options.enable_external_access) {
129
+ throw PermissionException("Installing extensions is disabled through configuration");
130
+ }
87
131
  auto extension_name = ApplyExtensionAlias(fs.ExtractBaseName(extension));
88
132
 
89
133
  string local_extension_path = fs.JoinPath(local_path, extension_name + ".duckdb_extension");
@@ -123,7 +167,7 @@ void ExtensionHelper::InstallExtension(ClientContext &context, const string &ext
123
167
 
124
168
  string default_endpoint = "http://extensions.duckdb.org";
125
169
  string versioned_path = "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension.gz";
126
- string &custom_endpoint = ClientConfig::GetConfig(context).custom_extension_repo;
170
+ string custom_endpoint = client_config ? client_config->custom_extension_repo : string();
127
171
  string &endpoint = !custom_endpoint.empty() ? custom_endpoint : default_endpoint;
128
172
  string url_template = endpoint + versioned_path;
129
173
 
@@ -22,7 +22,8 @@ static T LoadFunctionFromDLL(void *dll, const string &function_name, const strin
22
22
  return (T)function;
23
23
  }
24
24
 
25
- ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *opener, const string &extension) {
25
+ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileOpener *opener, const string &extension,
26
+ ExtensionInitResult &result, string &error) {
26
27
  if (!config.options.enable_external_access) {
27
28
  throw PermissionException("Loading external extensions is disabled through configuration");
28
29
  }
@@ -31,8 +32,14 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
31
32
  auto filename = fs.ConvertSeparators(extension);
32
33
 
33
34
  // shorthand case
34
- if (!StringUtil::Contains(extension, ".") && !StringUtil::Contains(extension, fs.PathSeparator())) {
35
- string local_path = fs.GetHomeDirectory(opener);
35
+ if (!ExtensionHelper::IsFullPath(extension)) {
36
+ string local_path = !config.options.extension_directory.empty() ? config.options.extension_directory
37
+ : fs.GetHomeDirectory(opener);
38
+
39
+ // convert random separators to platform-canonic
40
+ local_path = fs.ConvertSeparators(local_path);
41
+ // expand ~ in extension directory
42
+ local_path = fs.ExpandPath(local_path, opener);
36
43
  auto path_components = PathComponents();
37
44
  for (auto &path_ele : path_components) {
38
45
  local_path = fs.JoinPath(local_path, path_ele);
@@ -40,14 +47,14 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
40
47
  string extension_name = ApplyExtensionAlias(extension);
41
48
  filename = fs.JoinPath(local_path, extension_name + ".duckdb_extension");
42
49
  }
43
-
44
50
  if (!fs.FileExists(filename)) {
45
51
  string message;
46
52
  bool exact_match = ExtensionHelper::CreateSuggestions(extension, message);
47
53
  if (exact_match) {
48
54
  message += "\nInstall it first using \"INSTALL " + extension + "\".";
49
55
  }
50
- throw IOException("Extension \"%s\" not found.\n%s", filename, message);
56
+ error = StringUtil::Format("Extension \"%s\" not found.\n%s", filename, message);
57
+ return false;
51
58
  }
52
59
  if (!config.options.allow_unsigned_extensions) {
53
60
  auto handle = fs.OpenFile(filename, FileFlags::FILE_FLAGS_READ);
@@ -114,16 +121,43 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
114
121
  extension_version, engine_version);
115
122
  }
116
123
 
117
- ExtensionInitResult res;
118
- res.basename = basename;
119
- res.filename = filename;
120
- res.lib_hdl = lib_hdl;
121
- return res;
124
+ result.basename = basename;
125
+ result.filename = filename;
126
+ result.lib_hdl = lib_hdl;
127
+ return true;
128
+ }
129
+
130
+ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *opener, const string &extension) {
131
+ string error;
132
+ ExtensionInitResult result;
133
+ if (!TryInitialLoad(config, opener, extension, result, error)) {
134
+ throw IOException(error);
135
+ }
136
+ return result;
137
+ }
138
+
139
+ bool ExtensionHelper::IsFullPath(const string &extension) {
140
+ return StringUtil::Contains(extension, ".") || StringUtil::Contains(extension, "/") ||
141
+ StringUtil::Contains(extension, "\\");
142
+ }
143
+
144
+ string ExtensionHelper::GetExtensionName(const string &extension) {
145
+ if (!IsFullPath(extension)) {
146
+ return extension;
147
+ }
148
+ auto splits = StringUtil::Split(StringUtil::Replace(extension, "\\", "/"), '/');
149
+ if (splits.empty()) {
150
+ return extension;
151
+ }
152
+ splits = StringUtil::Split(splits.back(), '.');
153
+ if (splits.empty()) {
154
+ return extension;
155
+ }
156
+ return StringUtil::Lower(splits.front());
122
157
  }
123
158
 
124
159
  void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileOpener *opener, const string &extension) {
125
- auto &loaded_extensions = db.LoadedExtensions();
126
- if (loaded_extensions.find(extension) != loaded_extensions.end()) {
160
+ if (db.ExtensionIsLoaded(extension)) {
127
161
  return;
128
162
  }
129
163
 
@@ -149,7 +183,22 @@ void ExtensionHelper::LoadExternalExtension(ClientContext &context, const string
149
183
 
150
184
  void ExtensionHelper::StorageInit(string &extension, DBConfig &config) {
151
185
  extension = ExtensionHelper::ApplyExtensionAlias(extension);
152
- auto res = InitialLoad(config, nullptr, extension); // TODO opener
186
+ ExtensionInitResult res;
187
+ string error;
188
+ if (!TryInitialLoad(config, nullptr, extension, res, error)) {
189
+ if (!ExtensionHelper::AllowAutoInstall(extension)) {
190
+ throw IOException(error);
191
+ }
192
+ // the extension load failed - try installing the extension
193
+ if (!config.file_system) {
194
+ throw InternalException("Attempting to install an extension without a file system");
195
+ }
196
+ ExtensionHelper::InstallExtension(config, *config.file_system, extension, false);
197
+ // try loading again
198
+ if (!TryInitialLoad(config, nullptr, extension, res, error)) {
199
+ throw IOException(error);
200
+ }
201
+ }
153
202
  auto storage_fun_name = res.basename + "_storage_init";
154
203
 
155
204
  ext_storage_init_t storage_init_fun;
@@ -531,6 +531,22 @@ Value ExplainOutputSetting::GetSetting(ClientContext &context) {
531
531
  }
532
532
  }
533
533
 
534
+ //===--------------------------------------------------------------------===//
535
+ // Extension Directory Setting
536
+ //===--------------------------------------------------------------------===//
537
+ void ExtensionDirectorySetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
538
+ auto new_directory = input.ToString();
539
+ config.options.extension_directory = input.ToString();
540
+ }
541
+
542
+ void ExtensionDirectorySetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
543
+ config.options.extension_directory = DBConfig().options.extension_directory;
544
+ }
545
+
546
+ Value ExtensionDirectorySetting::GetSetting(ClientContext &context) {
547
+ return Value(DBConfig::GetConfig(context).options.extension_directory);
548
+ }
549
+
534
550
  //===--------------------------------------------------------------------===//
535
551
  // External Threads Setting
536
552
  //===--------------------------------------------------------------------===//
@@ -83,12 +83,8 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq
83
83
  *node_ptr = std::move(cross_product);
84
84
  return;
85
85
  }
86
- case JoinType::INNER:
87
- case JoinType::LEFT:
88
- case JoinType::RIGHT:
89
- case JoinType::OUTER: {
90
- // inner/left/right/full outer join, replace with cross product
91
- // since the condition is always true, left/right/outer join are equivalent to inner join here
86
+ case JoinType::INNER: {
87
+ // inner, replace with cross product
92
88
  auto cross_product =
93
89
  LogicalCrossProduct::Create(std::move(join.children[0]), std::move(join.children[1]));
94
90
  *node_ptr = std::move(cross_product);
@@ -114,52 +114,6 @@ OperatorResultType PipelineExecutor::ExecutePushInternal(DataChunk &input, idx_t
114
114
  }
115
115
  }
116
116
 
117
- // Pull a single DataChunk from the pipeline by flushing any operators holding cached output
118
- void PipelineExecutor::FlushCachingOperatorsPull(DataChunk &result) {
119
- idx_t start_idx = IsFinished() ? idx_t(finished_processing_idx) : 0;
120
- idx_t op_idx = start_idx;
121
- while (op_idx < pipeline.operators.size()) {
122
- if (!pipeline.operators[op_idx]->RequiresFinalExecute()) {
123
- op_idx++;
124
- continue;
125
- }
126
-
127
- OperatorFinalizeResultType finalize_result;
128
- DataChunk &curr_chunk =
129
- op_idx + 1 >= intermediate_chunks.size() ? final_chunk : *intermediate_chunks[op_idx + 1];
130
-
131
- if (pending_final_execute) {
132
- // Still have a cached chunk from a last pull, reuse chunk
133
- finalize_result = cached_final_execute_result;
134
- } else {
135
- // Flush the current operator
136
- auto current_operator = pipeline.operators[op_idx];
137
- StartOperator(current_operator);
138
- finalize_result = current_operator->FinalExecute(context, curr_chunk, *current_operator->op_state,
139
- *intermediate_states[op_idx]);
140
- EndOperator(current_operator, &curr_chunk);
141
- }
142
-
143
- auto execute_result = Execute(curr_chunk, result, op_idx + 1);
144
-
145
- if (execute_result == OperatorResultType::HAVE_MORE_OUTPUT) {
146
- pending_final_execute = true;
147
- cached_final_execute_result = finalize_result;
148
- } else {
149
- pending_final_execute = false;
150
- if (finalize_result == OperatorFinalizeResultType::FINISHED) {
151
- FinishProcessing(op_idx);
152
- op_idx++;
153
- }
154
- }
155
-
156
- // Some non-empty result was pulled from some caching operator, we're done for this pull
157
- if (result.size() > 0) {
158
- break;
159
- }
160
- }
161
- }
162
-
163
117
  // Push all remaining cached operator output through the pipeline
164
118
  void PipelineExecutor::FlushCachingOperatorsPush() {
165
119
  idx_t start_idx = IsFinished() ? idx_t(finished_processing_idx) : 0;
@@ -223,21 +177,13 @@ void PipelineExecutor::ExecutePull(DataChunk &result) {
223
177
  D_ASSERT(!pipeline.sink);
224
178
  auto &source_chunk = pipeline.operators.empty() ? result : *intermediate_chunks[0];
225
179
  while (result.size() == 0) {
226
- if (source_empty) {
227
- FlushCachingOperatorsPull(result);
228
- break;
229
- }
230
-
231
180
  if (in_process_operators.empty()) {
232
181
  source_chunk.Reset();
233
182
  FetchFromSource(source_chunk);
234
-
235
183
  if (source_chunk.size() == 0) {
236
- source_empty = true;
237
- continue;
184
+ break;
238
185
  }
239
186
  }
240
-
241
187
  if (!pipeline.operators.empty()) {
242
188
  auto state = Execute(source_chunk, result);
243
189
  if (state == OperatorResultType::FINISHED) {
@@ -16,6 +16,9 @@ unique_ptr<CreateInfo> CreateIndexInfo::Copy() const {
16
16
  for (auto &expr : expressions) {
17
17
  result->expressions.push_back(expr->Copy());
18
18
  }
19
+ for (auto &expr : parsed_expressions) {
20
+ result->parsed_expressions.push_back(expr->Copy());
21
+ }
19
22
 
20
23
  result->scan_types = scan_types;
21
24
  result->names = names;
@@ -11,16 +11,6 @@ CopyStatement::CopyStatement(const CopyStatement &other) : SQLStatement(other),
11
11
  }
12
12
  }
13
13
 
14
- string ConvertOptionValueToString(const Value &val) {
15
- auto type = val.type().id();
16
- switch (type) {
17
- case LogicalTypeId::VARCHAR:
18
- return KeywordHelper::WriteOptionallyQuoted(val.ToString());
19
- default:
20
- return val.ToString();
21
- }
22
- }
23
-
24
14
  string CopyStatement::CopyOptionsToString(const string &format,
25
15
  const case_insensitive_map_t<vector<Value>> &options) const {
26
16
  if (format.empty() && options.empty()) {
@@ -45,15 +35,14 @@ string CopyStatement::CopyOptionsToString(const string &format,
45
35
  // Options like HEADER don't need an explicit value
46
36
  // just providing the name already sets it to true
47
37
  } else if (values.size() == 1) {
48
- result += ConvertOptionValueToString(values[0]);
38
+ result += values[0].ToSQLString();
49
39
  } else {
50
40
  result += "( ";
51
41
  for (idx_t i = 0; i < values.size(); i++) {
52
- auto &value = values[i];
53
42
  if (i) {
54
43
  result += ", ";
55
44
  }
56
- result += KeywordHelper::WriteOptionallyQuoted(value.ToString());
45
+ result += values[i].ToSQLString();
57
46
  }
58
47
  result += " )";
59
48
  }
@@ -13,6 +13,9 @@ DeleteStatement::DeleteStatement(const DeleteStatement &other) : SQLStatement(ot
13
13
  for (const auto &using_clause : other.using_clauses) {
14
14
  using_clauses.push_back(using_clause->Copy());
15
15
  }
16
+ for (auto &expr : other.returning_list) {
17
+ returning_list.emplace_back(expr->Copy());
18
+ }
16
19
  cte_map = other.cte_map.Copy();
17
20
  }
18
21
 
@@ -13,6 +13,9 @@ OnConflictInfo::OnConflictInfo(const OnConflictInfo &other)
13
13
  if (other.set_info) {
14
14
  set_info = other.set_info->Copy();
15
15
  }
16
+ if (other.condition) {
17
+ condition = other.condition->Copy();
18
+ }
16
19
  }
17
20
 
18
21
  unique_ptr<OnConflictInfo> OnConflictInfo::Copy() const {
@@ -28,6 +31,12 @@ InsertStatement::InsertStatement(const InsertStatement &other)
28
31
  select_statement(unique_ptr_cast<SQLStatement, SelectStatement>(other.select_statement->Copy())),
29
32
  columns(other.columns), table(other.table), schema(other.schema), catalog(other.catalog) {
30
33
  cte_map = other.cte_map.Copy();
34
+ for (auto &expr : other.returning_list) {
35
+ returning_list.emplace_back(expr->Copy());
36
+ }
37
+ if (other.table_ref) {
38
+ table_ref = other.table_ref->Copy();
39
+ }
31
40
  if (other.on_conflict_info) {
32
41
  on_conflict_info = other.on_conflict_info->Copy();
33
42
  }
@@ -27,6 +27,9 @@ UpdateStatement::UpdateStatement(const UpdateStatement &other)
27
27
  if (other.from_table) {
28
28
  from_table = other.from_table->Copy();
29
29
  }
30
+ for (auto &expr : other.returning_list) {
31
+ returning_list.emplace_back(expr->Copy());
32
+ }
30
33
  cte_map = other.cte_map.Copy();
31
34
  }
32
35
 
@@ -9,16 +9,16 @@ unique_ptr<ParsedExpression> Transformer::TransformCase(duckdb_libpgquery::PGCas
9
9
  D_ASSERT(root);
10
10
 
11
11
  auto case_node = make_unique<CaseExpression>();
12
+ auto root_arg = TransformExpression(reinterpret_cast<duckdb_libpgquery::PGNode *>(root->arg));
12
13
  for (auto cell = root->args->head; cell != nullptr; cell = cell->next) {
13
14
  CaseCheck case_check;
14
15
 
15
16
  auto w = reinterpret_cast<duckdb_libpgquery::PGCaseWhen *>(cell->data.ptr_value);
16
17
  auto test_raw = TransformExpression(reinterpret_cast<duckdb_libpgquery::PGNode *>(w->expr));
17
18
  unique_ptr<ParsedExpression> test;
18
- auto arg = TransformExpression(reinterpret_cast<duckdb_libpgquery::PGNode *>(root->arg));
19
- if (arg) {
19
+ if (root_arg) {
20
20
  case_check.when_expr =
21
- make_unique<ComparisonExpression>(ExpressionType::COMPARE_EQUAL, std::move(arg), std::move(test_raw));
21
+ make_unique<ComparisonExpression>(ExpressionType::COMPARE_EQUAL, root_arg->Copy(), std::move(test_raw));
22
22
  } else {
23
23
  case_check.when_expr = std::move(test_raw);
24
24
  }
@@ -44,7 +44,7 @@ vector<string> BindContext::GetSimilarBindings(const string &column_name) {
44
44
  for (auto &kv : bindings) {
45
45
  auto binding = kv.second.get();
46
46
  for (auto &name : binding->names) {
47
- idx_t distance = StringUtil::LevenshteinDistance(name, column_name);
47
+ idx_t distance = StringUtil::SimilarityScore(name, column_name);
48
48
  scores.emplace_back(binding->alias + "." + name, distance);
49
49
  }
50
50
  }
@@ -72,6 +72,9 @@ static void NegatePercentileFractions(ClientContext &context, unique_ptr<ParsedE
72
72
  for (const auto &element_val : ListValue::GetChildren(value)) {
73
73
  values.push_back(NegatePercentileValue(element_val, desc));
74
74
  }
75
+ if (values.empty()) {
76
+ throw BinderException("Empty list in percentile not allowed");
77
+ }
75
78
  bound.expr = make_unique<BoundConstantExpression>(Value::LIST(values));
76
79
  } else {
77
80
  bound.expr = make_unique<BoundConstantExpression>(NegatePercentileValue(value, desc));
@@ -23,16 +23,6 @@
23
23
 
24
24
  namespace duckdb {
25
25
 
26
- static vector<idx_t> ColumnListToIndices(const vector<bool> &vec) {
27
- vector<idx_t> ret;
28
- for (idx_t i = 0; i < vec.size(); i++) {
29
- if (vec[i]) {
30
- ret.push_back(i);
31
- }
32
- }
33
- return ret;
34
- }
35
-
36
26
  vector<string> GetUniqueNames(const vector<string> &original_names) {
37
27
  unordered_set<string> name_set;
38
28
  vector<string> unique_names;
@@ -97,22 +87,25 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
97
87
  for (auto &option : original_options) {
98
88
  auto loption = StringUtil::Lower(option.first);
99
89
  if (loption == "use_tmp_file") {
100
- use_tmp_file = option.second[0].CastAs(context, LogicalType::BOOLEAN).GetValue<bool>();
90
+ use_tmp_file =
91
+ option.second.empty() || option.second[0].CastAs(context, LogicalType::BOOLEAN).GetValue<bool>();
101
92
  user_set_use_tmp_file = true;
102
93
  continue;
103
94
  }
104
95
  if (loption == "allow_overwrite") {
105
- allow_overwrite = option.second[0].CastAs(context, LogicalType::BOOLEAN).GetValue<bool>();
96
+ allow_overwrite =
97
+ option.second.empty() || option.second[0].CastAs(context, LogicalType::BOOLEAN).GetValue<bool>();
106
98
  continue;
107
99
  }
108
100
 
109
101
  if (loption == "per_thread_output") {
110
- per_thread_output = option.second[0].CastAs(context, LogicalType::BOOLEAN).GetValue<bool>();
102
+ per_thread_output =
103
+ option.second.empty() || option.second[0].CastAs(context, LogicalType::BOOLEAN).GetValue<bool>();
111
104
  continue;
112
105
  }
113
106
  if (loption == "partition_by") {
114
107
  auto converted = ConvertVectorToValue(std::move(option.second));
115
- partition_cols = ColumnListToIndices(ParseColumnList(converted, select_node.names, loption));
108
+ partition_cols = ParseColumnsOrdered(converted, select_node.names, loption);
116
109
  continue;
117
110
  }
118
111
  stmt.info->options[option.first] = option.second;
@@ -16,6 +16,8 @@
16
16
  #include "duckdb/parser/expression/list.hpp"
17
17
  #include "duckdb/common/index_map.hpp"
18
18
  #include "duckdb/planner/expression_iterator.hpp"
19
+ #include "duckdb/planner/expression_binder/index_binder.hpp"
20
+ #include "duckdb/parser/parsed_data/create_index_info.hpp"
19
21
 
20
22
  #include <algorithm>
21
23
 
@@ -300,4 +302,15 @@ unique_ptr<BoundCreateTableInfo> Binder::BindCreateTableInfo(unique_ptr<CreateIn
300
302
  return BindCreateTableInfo(std::move(info), schema);
301
303
  }
302
304
 
305
+ vector<unique_ptr<Expression>> Binder::BindCreateIndexExpressions(TableCatalogEntry *table, CreateIndexInfo *info) {
306
+ vector<unique_ptr<Expression>> expressions;
307
+
308
+ auto index_binder = IndexBinder(*this, this->context, table, info);
309
+ for (auto &expr : info->expressions) {
310
+ expressions.push_back(index_binder.Bind(expr));
311
+ }
312
+
313
+ return expressions;
314
+ }
315
+
303
316
  } // namespace duckdb
@@ -54,7 +54,7 @@ BoundStatement Binder::Bind(DropStatement &stmt) {
54
54
  auto &config = DBConfig::GetConfig(context);
55
55
  // for now assume only one storage extension provides the custom drop_database impl
56
56
  for (auto &extension_entry : config.storage_extensions) {
57
- if (extension_entry.second->drop_database != nullptr) {
57
+ if (extension_entry.second->drop_database == nullptr) {
58
58
  continue;
59
59
  }
60
60
  auto &storage_extension = extension_entry.second;
@@ -64,7 +64,7 @@ BoundStatement Binder::Bind(DropStatement &stmt) {
64
64
  auto bound_drop_database_func = Bind(*drop_database_function_ref);
65
65
  result.plan = CreatePlan(*bound_drop_database_func);
66
66
  result.names = {"Success"};
67
- result.types = {LogicalType::BOOLEAN};
67
+ result.types = {LogicalType::BIGINT};
68
68
  properties.allow_stream_result = false;
69
69
  properties.return_type = StatementReturnType::NOTHING;
70
70
  return result;
@@ -301,7 +301,28 @@ void Binder::BindOnConflictClause(LogicalInsert &insert, TableCatalogEntry &tabl
301
301
  insert.on_conflict_condition = std::move(condition);
302
302
  }
303
303
 
304
- auto projection_index = insert.children[0]->GetTableIndex()[0];
304
+ auto bindings = insert.children[0]->GetColumnBindings();
305
+ idx_t projection_index = DConstants::INVALID_INDEX;
306
+ std::vector<unique_ptr<LogicalOperator>> *insert_child_operators;
307
+ insert_child_operators = &insert.children;
308
+ while (projection_index == DConstants::INVALID_INDEX) {
309
+ if (insert_child_operators->empty()) {
310
+ // No further children to visit
311
+ break;
312
+ }
313
+ D_ASSERT(insert_child_operators->size() >= 1);
314
+ auto &current_child = (*insert_child_operators)[0];
315
+ auto table_indices = current_child->GetTableIndex();
316
+ if (table_indices.empty()) {
317
+ // This operator does not have a table index to refer to, we have to visit its children
318
+ insert_child_operators = &current_child->children;
319
+ continue;
320
+ }
321
+ projection_index = table_indices[0];
322
+ }
323
+ if (projection_index == DConstants::INVALID_INDEX) {
324
+ throw InternalException("Could not locate a table_index from the children of the insert");
325
+ }
305
326
 
306
327
  string unused;
307
328
  auto original_binding = bind_context.GetBinding(table_alias, unused);