duckdb 0.8.2-dev4474.0 → 0.8.2-dev4572.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/lib/duckdb.js +11 -1
  2. package/package.json +3 -1
  3. package/src/connection.cpp +48 -7
  4. package/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp +3 -0
  5. package/src/duckdb/src/catalog/catalog.cpp +5 -0
  6. package/src/duckdb/src/catalog/duck_catalog.cpp +4 -0
  7. package/src/duckdb/src/execution/column_binding_resolver.cpp +1 -0
  8. package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +59 -38
  9. package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
  10. package/src/duckdb/src/function/table/arrow.cpp +18 -13
  11. package/src/duckdb/src/function/table/system/pragma_metadata_info.cpp +83 -0
  12. package/src/duckdb/src/function/table/system/pragma_storage_info.cpp +5 -0
  13. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  14. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  15. package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +2 -0
  16. package/src/duckdb/src/include/duckdb/catalog/duck_catalog.hpp +1 -0
  17. package/src/duckdb/src/include/duckdb/common/serializer/deserialization_data.hpp +36 -0
  18. package/src/duckdb/src/include/duckdb/function/compression_function.hpp +36 -4
  19. package/src/duckdb/src/include/duckdb/function/table/arrow.hpp +2 -0
  20. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
  21. package/src/duckdb/src/include/duckdb/optimizer/rule/empty_needle_removal.hpp +1 -1
  22. package/src/duckdb/src/include/duckdb/storage/checkpoint/string_checkpoint_state.hpp +27 -4
  23. package/src/duckdb/src/include/duckdb/storage/checkpoint/write_overflow_strings_to_disk.hpp +4 -2
  24. package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp +22 -1
  25. package/src/duckdb/src/include/duckdb/storage/database_size.hpp +6 -0
  26. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +2 -0
  27. package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +2 -0
  28. package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp +6 -1
  29. package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp +7 -3
  30. package/src/duckdb/src/include/duckdb/storage/table_storage_info.hpp +1 -0
  31. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +9 -3
  32. package/src/duckdb/src/optimizer/statistics/operator/propagate_projection.cpp +0 -1
  33. package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp +1 -4
  34. package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp +1 -4
  35. package/src/duckdb/src/storage/checkpoint/write_overflow_strings_to_disk.cpp +47 -10
  36. package/src/duckdb/src/storage/checkpoint_manager.cpp +0 -2
  37. package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp +6 -1
  38. package/src/duckdb/src/storage/compression/string_uncompressed.cpp +62 -12
  39. package/src/duckdb/src/storage/compression/validity_uncompressed.cpp +2 -1
  40. package/src/duckdb/src/storage/data_pointer.cpp +20 -0
  41. package/src/duckdb/src/storage/local_storage.cpp +3 -7
  42. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +29 -15
  43. package/src/duckdb/src/storage/serialization/serialize_storage.cpp +4 -0
  44. package/src/duckdb/src/storage/single_file_block_manager.cpp +15 -9
  45. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  46. package/src/duckdb/src/storage/storage_manager.cpp +5 -0
  47. package/src/duckdb/src/storage/table/column_checkpoint_state.cpp +3 -0
  48. package/src/duckdb/src/storage/table/column_data.cpp +17 -14
  49. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +4 -8
  50. package/src/duckdb/src/storage/table/column_segment.cpp +21 -12
  51. package/src/duckdb/ub_src_function_table_system.cpp +2 -0
  52. package/src/duckdb/ub_src_storage.cpp +2 -0
  53. package/src/duckdb_node.hpp +1 -0
  54. package/test/close_hang.test.ts +39 -0
package/lib/duckdb.js CHANGED
@@ -412,6 +412,13 @@ Connection.prototype.register_buffer;
412
412
  */
413
413
  Connection.prototype.unregister_buffer;
414
414
 
415
+ /**
416
+ * Closes connection
417
+ * @method
418
+ * @param callback
419
+ * @return {void}
420
+ */
421
+ Connection.prototype.close;
415
422
 
416
423
  /**
417
424
  * Closes database instance
@@ -420,7 +427,10 @@ Connection.prototype.unregister_buffer;
420
427
  * @return {void}
421
428
  */
422
429
  Database.prototype.close = function() {
423
- this.default_connection = null
430
+ if (this.default_connection) {
431
+ this.default_connection.close(); // this queues up a job in the internals, which blocks the below close call
432
+ this.default_connection = null;
433
+ }
424
434
  this.close_internal.apply(this, arguments);
425
435
  };
426
436
 
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "duckdb",
3
3
  "main": "./lib/duckdb.js",
4
4
  "types": "./lib/duckdb.d.ts",
5
- "version": "0.8.2-dev4474.0",
5
+ "version": "0.8.2-dev4572.0",
6
6
  "description": "DuckDB node.js API",
7
7
  "gypfile": true,
8
8
  "dependencies": {
@@ -29,12 +29,14 @@
29
29
  "devDependencies": {
30
30
  "@types/chai": "^4.3.4",
31
31
  "@types/chai-as-promised": "^7.1.5",
32
+ "@types/fs-extra": "^11.0.1",
32
33
  "@types/mocha": "^10.0.0",
33
34
  "@types/node": "^18.11.0",
34
35
  "apache-arrow": "^9.0.0",
35
36
  "aws-sdk": "^2.790.0",
36
37
  "chai": "^4.3.6",
37
38
  "chai-as-promised": "^7.1.1",
39
+ "fs-extra": "^11.1.1",
38
40
  "jsdoc3-parser": "^2.0.0",
39
41
  "mocha": "^8.3.0",
40
42
  "ts-node": "^10.9.1",
@@ -12,13 +12,13 @@ namespace node_duckdb {
12
12
  Napi::FunctionReference Connection::Init(Napi::Env env, Napi::Object exports) {
13
13
  Napi::HandleScope scope(env);
14
14
 
15
- Napi::Function t =
16
- DefineClass(env, "Connection",
17
- {InstanceMethod("prepare", &Connection::Prepare), InstanceMethod("exec", &Connection::Exec),
18
- InstanceMethod("register_udf_bulk", &Connection::RegisterUdf),
19
- InstanceMethod("register_buffer", &Connection::RegisterBuffer),
20
- InstanceMethod("unregister_udf", &Connection::UnregisterUdf),
21
- InstanceMethod("unregister_buffer", &Connection::UnRegisterBuffer)});
15
+ Napi::Function t = DefineClass(
16
+ env, "Connection",
17
+ {InstanceMethod("prepare", &Connection::Prepare), InstanceMethod("exec", &Connection::Exec),
18
+ InstanceMethod("register_udf_bulk", &Connection::RegisterUdf),
19
+ InstanceMethod("register_buffer", &Connection::RegisterBuffer),
20
+ InstanceMethod("unregister_udf", &Connection::UnregisterUdf), InstanceMethod("close", &Connection::Close),
21
+ InstanceMethod("unregister_buffer", &Connection::UnRegisterBuffer)});
22
22
 
23
23
  exports.Set("Connection", t);
24
24
 
@@ -407,6 +407,36 @@ struct ExecTaskWithCallback : public ExecTask {
407
407
  std::function<void(void)> cpp_callback;
408
408
  };
409
409
 
410
+ struct CloseConnectionTask : public Task {
411
+ CloseConnectionTask(Connection &connection, Napi::Function callback) : Task(connection, callback) {
412
+ }
413
+
414
+ void DoWork() override {
415
+ auto &connection = Get<Connection>();
416
+ if (connection.connection) {
417
+ connection.connection.reset();
418
+ success = true;
419
+ } else {
420
+ success = false;
421
+ }
422
+ }
423
+
424
+ void Callback() override {
425
+ auto &connection = Get<Connection>();
426
+ auto env = connection.Env();
427
+ Napi::HandleScope scope(env);
428
+
429
+ auto cb = callback.Value();
430
+ if (!success) {
431
+ cb.MakeCallback(connection.Value(), {Utils::CreateError(env, "Connection was already closed")});
432
+ return;
433
+ }
434
+ cb.MakeCallback(connection.Value(), {env.Null(), connection.Value()});
435
+ }
436
+
437
+ bool success = false;
438
+ };
439
+
410
440
  Napi::Value Connection::Exec(const Napi::CallbackInfo &info) {
411
441
  auto env = info.Env();
412
442
 
@@ -512,6 +542,17 @@ Napi::Value Connection::UnRegisterBuffer(const Napi::CallbackInfo &info) {
512
542
  return Value();
513
543
  }
514
544
 
545
+ Napi::Value Connection::Close(const Napi::CallbackInfo &info) {
546
+ Napi::Function callback;
547
+ if (info.Length() > 0 && info[0].IsFunction()) {
548
+ callback = info[0].As<Napi::Function>();
549
+ }
550
+
551
+ database_ref->Schedule(info.Env(), duckdb::make_uniq<CloseConnectionTask>(*this, callback));
552
+
553
+ return info.Env().Undefined();
554
+ }
555
+
515
556
  Napi::Object Connection::NewInstance(const Napi::Value &db) {
516
557
  return NodeDuckDB::GetData(db.Env())->connection_constructor.New({db});
517
558
  }
@@ -244,6 +244,9 @@ struct ExecuteSqlTableFunction {
244
244
  auto result = make_uniq<BindData>();
245
245
 
246
246
  result->con = make_uniq<Connection>(*context.db);
247
+ if (input.inputs[0].IsNull()) {
248
+ throw BinderException("json_execute_serialized_sql cannot execute NULL plan");
249
+ }
247
250
  auto serialized = input.inputs[0].GetValueUnsafe<string>();
248
251
  auto stmt = DeserializeSelectStatement(serialized, alc);
249
252
  result->plan = result->con->RelationFromQuery(std::move(stmt));
@@ -35,6 +35,7 @@
35
35
  #include "duckdb/main/database_manager.hpp"
36
36
  #include "duckdb/function/built_in_functions.hpp"
37
37
  #include "duckdb/catalog/similar_catalog_entry.hpp"
38
+ #include "duckdb/storage/database_size.hpp"
38
39
  #include <algorithm>
39
40
 
40
41
  namespace duckdb {
@@ -831,6 +832,10 @@ void Catalog::Alter(ClientContext &context, AlterInfo &info) {
831
832
  return lookup.schema->Alter(context, info);
832
833
  }
833
834
 
835
+ vector<MetadataBlockInfo> Catalog::GetMetadataInfo(ClientContext &context) {
836
+ return vector<MetadataBlockInfo>();
837
+ }
838
+
834
839
  void Catalog::Verify() {
835
840
  }
836
841
 
@@ -132,6 +132,10 @@ DatabaseSize DuckCatalog::GetDatabaseSize(ClientContext &context) {
132
132
  return db.GetStorageManager().GetDatabaseSize();
133
133
  }
134
134
 
135
+ vector<MetadataBlockInfo> DuckCatalog::GetMetadataInfo(ClientContext &context) {
136
+ return db.GetStorageManager().GetMetadataInfo();
137
+ }
138
+
135
139
  bool DuckCatalog::InMemory() {
136
140
  return db.GetStorageManager().InMemory();
137
141
  }
@@ -102,6 +102,7 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
102
102
  default:
103
103
  break;
104
104
  }
105
+
105
106
  // general case
106
107
  // first visit the children of this operator
107
108
  VisitOperatorChildren(op);
@@ -6,6 +6,7 @@
6
6
  #include "duckdb/storage/table_io_manager.hpp"
7
7
  #include "duckdb/transaction/local_storage.hpp"
8
8
  #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
9
+ #include "duckdb/transaction/duck_transaction.hpp"
9
10
  #include "duckdb/storage/table/append_state.hpp"
10
11
  #include "duckdb/storage/table/scan_state.hpp"
11
12
 
@@ -119,6 +120,7 @@ public:
119
120
  idx_t insert_count;
120
121
  vector<RowGroupBatchEntry> collections;
121
122
  idx_t next_start = 0;
123
+ bool optimistically_written = false;
122
124
 
123
125
  void FindMergeCollections(idx_t min_batch_index, optional_idx &merged_batch_index,
124
126
  vector<unique_ptr<RowGroupCollection>> &result) {
@@ -176,10 +178,12 @@ public:
176
178
  unique_ptr<RowGroupCollection> MergeCollections(ClientContext &context,
177
179
  vector<unique_ptr<RowGroupCollection>> merge_collections,
178
180
  OptimisticDataWriter &writer) {
181
+ D_ASSERT(!merge_collections.empty());
179
182
  CollectionMerger merger(context);
180
183
  for (auto &collection : merge_collections) {
181
184
  merger.AddCollection(std::move(collection));
182
185
  }
186
+ optimistically_written = true;
183
187
  return merger.Flush(writer);
184
188
  }
185
189
 
@@ -373,48 +377,65 @@ SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event,
373
377
  OperatorSinkFinalizeInput &input) const {
374
378
  auto &gstate = input.global_state.Cast<BatchInsertGlobalState>();
375
379
 
376
- // in the finalize, do a final pass over all of the collections we created and try to merge smaller collections
377
- // together
378
- vector<unique_ptr<CollectionMerger>> mergers;
379
- unique_ptr<CollectionMerger> current_merger;
380
-
381
- auto &storage = gstate.table.GetStorage();
382
- for (auto &entry : gstate.collections) {
383
- if (entry.type == RowGroupBatchType::NOT_FLUSHED) {
384
- // this collection has not been flushed: add it to the merge set
385
- if (!current_merger) {
386
- current_merger = make_uniq<CollectionMerger>(context);
387
- }
388
- current_merger->AddCollection(std::move(entry.collection));
389
- } else {
390
- // this collection has been flushed: it does not need to be merged
391
- // create a separate collection merger only for this entry
392
- if (current_merger) {
393
- // we have small collections remaining: flush them
394
- mergers.push_back(std::move(current_merger));
395
- current_merger.reset();
380
+ if (gstate.optimistically_written || gstate.insert_count >= LocalStorage::MERGE_THRESHOLD) {
381
+ // we have written data to disk optimistically or are inserting a large amount of data
382
+ // perform a final pass over all of the row groups and merge them together
383
+ vector<unique_ptr<CollectionMerger>> mergers;
384
+ unique_ptr<CollectionMerger> current_merger;
385
+
386
+ auto &storage = gstate.table.GetStorage();
387
+ for (auto &entry : gstate.collections) {
388
+ if (entry.type == RowGroupBatchType::NOT_FLUSHED) {
389
+ // this collection has not been flushed: add it to the merge set
390
+ if (!current_merger) {
391
+ current_merger = make_uniq<CollectionMerger>(context);
392
+ }
393
+ current_merger->AddCollection(std::move(entry.collection));
394
+ } else {
395
+ // this collection has been flushed: it does not need to be merged
396
+ // create a separate collection merger only for this entry
397
+ if (current_merger) {
398
+ // we have small collections remaining: flush them
399
+ mergers.push_back(std::move(current_merger));
400
+ current_merger.reset();
401
+ }
402
+ auto larger_merger = make_uniq<CollectionMerger>(context);
403
+ larger_merger->AddCollection(std::move(entry.collection));
404
+ mergers.push_back(std::move(larger_merger));
396
405
  }
397
- auto larger_merger = make_uniq<CollectionMerger>(context);
398
- larger_merger->AddCollection(std::move(entry.collection));
399
- mergers.push_back(std::move(larger_merger));
400
406
  }
401
- }
402
- if (current_merger) {
403
- mergers.push_back(std::move(current_merger));
404
- }
407
+ if (current_merger) {
408
+ mergers.push_back(std::move(current_merger));
409
+ }
405
410
 
406
- // now that we have created all of the mergers, perform the actual merging
407
- vector<unique_ptr<RowGroupCollection>> final_collections;
408
- final_collections.reserve(mergers.size());
409
- auto &writer = storage.CreateOptimisticWriter(context);
410
- for (auto &merger : mergers) {
411
- final_collections.push_back(merger->Flush(writer));
412
- }
413
- storage.FinalizeOptimisticWriter(context, writer);
411
+ // now that we have created all of the mergers, perform the actual merging
412
+ vector<unique_ptr<RowGroupCollection>> final_collections;
413
+ final_collections.reserve(mergers.size());
414
+ auto &writer = storage.CreateOptimisticWriter(context);
415
+ for (auto &merger : mergers) {
416
+ final_collections.push_back(merger->Flush(writer));
417
+ }
418
+ storage.FinalizeOptimisticWriter(context, writer);
414
419
 
415
- // finally, merge the row groups into the local storage
416
- for (auto &collection : final_collections) {
417
- storage.LocalMerge(context, *collection);
420
+ // finally, merge the row groups into the local storage
421
+ for (auto &collection : final_collections) {
422
+ storage.LocalMerge(context, *collection);
423
+ }
424
+ } else {
425
+ // we are writing a small amount of data to disk
426
+ // append directly to transaction local storage
427
+ auto &table = gstate.table;
428
+ auto &storage = table.GetStorage();
429
+ LocalAppendState append_state;
430
+ storage.InitializeLocalAppend(append_state, context);
431
+ auto &transaction = DuckTransaction::Get(context, table.catalog);
432
+ for (auto &entry : gstate.collections) {
433
+ entry.collection->Scan(transaction, [&](DataChunk &insert_chunk) {
434
+ storage.LocalAppend(append_state, table, context, insert_chunk);
435
+ return true;
436
+ });
437
+ }
438
+ storage.FinalizeLocalAppend(append_state);
418
439
  }
419
440
  return SinkFinalizeType::READY;
420
441
  }
@@ -187,9 +187,14 @@ string PragmaStorageInfo(ClientContext &context, const FunctionParameters &param
187
187
  return StringUtil::Format("SELECT * FROM pragma_storage_info('%s');", parameters.values[0].ToString());
188
188
  }
189
189
 
190
+ string PragmaMetadataInfo(ClientContext &context, const FunctionParameters &parameters) {
191
+ return "SELECT * FROM pragma_metadata_info();";
192
+ }
193
+
190
194
  void PragmaQueries::RegisterFunction(BuiltinFunctions &set) {
191
195
  set.AddFunction(PragmaFunction::PragmaCall("table_info", PragmaTableInfo, {LogicalType::VARCHAR}));
192
196
  set.AddFunction(PragmaFunction::PragmaCall("storage_info", PragmaStorageInfo, {LogicalType::VARCHAR}));
197
+ set.AddFunction(PragmaFunction::PragmaCall("metadata_info", PragmaMetadataInfo, {}));
193
198
  set.AddFunction(PragmaFunction::PragmaStatement("show_tables", PragmaShowTables));
194
199
  set.AddFunction(PragmaFunction::PragmaStatement("show_tables_expanded", PragmaShowTablesExpanded));
195
200
  set.AddFunction(PragmaFunction::PragmaStatement("show_databases", PragmaShowDatabases));
@@ -208,18 +208,10 @@ void ArrowTableFunction::RenameArrowColumns(vector<string> &names) {
208
208
  }
209
209
  }
210
210
 
211
- unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBind(ClientContext &context, TableFunctionBindInput &input,
212
- vector<LogicalType> &return_types, vector<string> &names) {
213
- auto stream_factory_ptr = input.inputs[0].GetPointer();
214
- auto stream_factory_produce = (stream_factory_produce_t)input.inputs[1].GetPointer(); // NOLINT
215
- auto stream_factory_get_schema = (stream_factory_get_schema_t)input.inputs[2].GetPointer(); // NOLINT
216
-
217
- auto res = make_uniq<ArrowScanFunctionData>(stream_factory_produce, stream_factory_ptr);
218
-
219
- auto &data = *res;
220
- stream_factory_get_schema(stream_factory_ptr, data.schema_root);
221
- for (idx_t col_idx = 0; col_idx < (idx_t)data.schema_root.arrow_schema.n_children; col_idx++) {
222
- auto &schema = *data.schema_root.arrow_schema.children[col_idx];
211
+ void ArrowTableFunction::PopulateArrowTableType(ArrowTableType &arrow_table, ArrowSchemaWrapper &schema_p,
212
+ vector<string> &names, vector<LogicalType> &return_types) {
213
+ for (idx_t col_idx = 0; col_idx < (idx_t)schema_p.arrow_schema.n_children; col_idx++) {
214
+ auto &schema = *schema_p.arrow_schema.children[col_idx];
223
215
  if (!schema.release) {
224
216
  throw InvalidInputException("arrow_scan: released schema passed");
225
217
  }
@@ -233,7 +225,7 @@ unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBind(ClientContext &contex
233
225
  } else {
234
226
  return_types.emplace_back(arrow_type->GetDuckType());
235
227
  }
236
- res->arrow_table.AddColumn(col_idx, std::move(arrow_type));
228
+ arrow_table.AddColumn(col_idx, std::move(arrow_type));
237
229
  auto format = string(schema.format);
238
230
  auto name = string(schema.name);
239
231
  if (name.empty()) {
@@ -241,6 +233,19 @@ unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBind(ClientContext &contex
241
233
  }
242
234
  names.push_back(name);
243
235
  }
236
+ }
237
+
238
+ unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBind(ClientContext &context, TableFunctionBindInput &input,
239
+ vector<LogicalType> &return_types, vector<string> &names) {
240
+ auto stream_factory_ptr = input.inputs[0].GetPointer();
241
+ auto stream_factory_produce = (stream_factory_produce_t)input.inputs[1].GetPointer(); // NOLINT
242
+ auto stream_factory_get_schema = (stream_factory_get_schema_t)input.inputs[2].GetPointer(); // NOLINT
243
+
244
+ auto res = make_uniq<ArrowScanFunctionData>(stream_factory_produce, stream_factory_ptr);
245
+
246
+ auto &data = *res;
247
+ stream_factory_get_schema(stream_factory_ptr, data.schema_root);
248
+ PopulateArrowTableType(res->arrow_table, data.schema_root, names, return_types);
244
249
  RenameArrowColumns(names);
245
250
  res->all_types = return_types;
246
251
  return std::move(res);
@@ -0,0 +1,83 @@
1
+ #include "duckdb/function/table/system_functions.hpp"
2
+
3
+ #include "duckdb/catalog/catalog.hpp"
4
+ #include "duckdb/storage/database_size.hpp"
5
+ #include "duckdb/main/database_manager.hpp"
6
+ #include "duckdb/function/function_set.hpp"
7
+ namespace duckdb {
8
+
9
+ struct PragmaMetadataFunctionData : public TableFunctionData {
10
+ explicit PragmaMetadataFunctionData() {
11
+ }
12
+
13
+ vector<MetadataBlockInfo> metadata_info;
14
+ };
15
+
16
+ struct PragmaMetadataOperatorData : public GlobalTableFunctionState {
17
+ PragmaMetadataOperatorData() : offset(0) {
18
+ }
19
+
20
+ idx_t offset;
21
+ };
22
+
23
+ static unique_ptr<FunctionData> PragmaMetadataInfoBind(ClientContext &context, TableFunctionBindInput &input,
24
+ vector<LogicalType> &return_types, vector<string> &names) {
25
+ names.emplace_back("block_id");
26
+ return_types.emplace_back(LogicalType::BIGINT);
27
+
28
+ names.emplace_back("total_blocks");
29
+ return_types.emplace_back(LogicalType::BIGINT);
30
+
31
+ names.emplace_back("free_blocks");
32
+ return_types.emplace_back(LogicalType::BIGINT);
33
+
34
+ names.emplace_back("free_list");
35
+ return_types.emplace_back(LogicalType::LIST(LogicalType::BIGINT));
36
+
37
+ string db_name =
38
+ input.inputs.empty() ? DatabaseManager::GetDefaultDatabase(context) : StringValue::Get(input.inputs[0]);
39
+ auto &catalog = Catalog::GetCatalog(context, db_name);
40
+ auto result = make_uniq<PragmaMetadataFunctionData>();
41
+ result->metadata_info = catalog.GetMetadataInfo(context);
42
+ return std::move(result);
43
+ }
44
+
45
+ unique_ptr<GlobalTableFunctionState> PragmaMetadataInfoInit(ClientContext &context, TableFunctionInitInput &input) {
46
+ return make_uniq<PragmaMetadataOperatorData>();
47
+ }
48
+
49
+ static void PragmaMetadataInfoFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
50
+ auto &bind_data = data_p.bind_data->Cast<PragmaMetadataFunctionData>();
51
+ auto &data = data_p.global_state->Cast<PragmaMetadataOperatorData>();
52
+ idx_t count = 0;
53
+ while (data.offset < bind_data.metadata_info.size() && count < STANDARD_VECTOR_SIZE) {
54
+ auto &entry = bind_data.metadata_info[data.offset++];
55
+
56
+ idx_t col_idx = 0;
57
+ // block_id
58
+ output.SetValue(col_idx++, count, Value::BIGINT(entry.block_id));
59
+ // total_blocks
60
+ output.SetValue(col_idx++, count, Value::BIGINT(entry.total_blocks));
61
+ // free_blocks
62
+ output.SetValue(col_idx++, count, Value::BIGINT(entry.free_list.size()));
63
+ // free_list
64
+ vector<Value> list_values;
65
+ for (auto &free_id : entry.free_list) {
66
+ list_values.push_back(Value::BIGINT(free_id));
67
+ }
68
+ output.SetValue(col_idx++, count, Value::LIST(LogicalType::BIGINT, std::move(list_values)));
69
+ count++;
70
+ }
71
+ output.SetCardinality(count);
72
+ }
73
+
74
+ void PragmaMetadataInfo::RegisterFunction(BuiltinFunctions &set) {
75
+ TableFunctionSet metadata_info("pragma_metadata_info");
76
+ metadata_info.AddFunction(
77
+ TableFunction({}, PragmaMetadataInfoFunction, PragmaMetadataInfoBind, PragmaMetadataInfoInit));
78
+ metadata_info.AddFunction(TableFunction({LogicalType::VARCHAR}, PragmaMetadataInfoFunction, PragmaMetadataInfoBind,
79
+ PragmaMetadataInfoInit));
80
+ set.AddFunction(metadata_info);
81
+ }
82
+
83
+ } // namespace duckdb
@@ -76,6 +76,9 @@ static unique_ptr<FunctionData> PragmaStorageInfoBind(ClientContext &context, Ta
76
76
  names.emplace_back("block_offset");
77
77
  return_types.emplace_back(LogicalType::BIGINT);
78
78
 
79
+ names.emplace_back("segment_info");
80
+ return_types.emplace_back(LogicalType::VARCHAR);
81
+
79
82
  auto qname = QualifiedName::Parse(input.inputs[0].GetValue<string>());
80
83
 
81
84
  // look up the table name in the catalog
@@ -133,6 +136,8 @@ static void PragmaStorageInfoFunction(ClientContext &context, TableFunctionInput
133
136
  output.SetValue(col_idx++, count, Value());
134
137
  output.SetValue(col_idx++, count, Value());
135
138
  }
139
+ // segment_info
140
+ output.SetValue(col_idx++, count, Value(entry.segment_info));
136
141
  count++;
137
142
  }
138
143
  output.SetCardinality(count);
@@ -14,6 +14,7 @@ void BuiltinFunctions::RegisterSQLiteFunctions() {
14
14
  PragmaCollations::RegisterFunction(*this);
15
15
  PragmaTableInfo::RegisterFunction(*this);
16
16
  PragmaStorageInfo::RegisterFunction(*this);
17
+ PragmaMetadataInfo::RegisterFunction(*this);
17
18
  PragmaDatabaseSize::RegisterFunction(*this);
18
19
  PragmaLastProfilingOutput::RegisterFunction(*this);
19
20
  PragmaDetailedProfilingOutput::RegisterFunction(*this);
@@ -1,8 +1,8 @@
1
1
  #ifndef DUCKDB_VERSION
2
- #define DUCKDB_VERSION "0.8.2-dev4474"
2
+ #define DUCKDB_VERSION "0.8.2-dev4572"
3
3
  #endif
4
4
  #ifndef DUCKDB_SOURCE_ID
5
- #define DUCKDB_SOURCE_ID "ba71015ee7"
5
+ #define DUCKDB_SOURCE_ID "53dc13de5c"
6
6
  #endif
7
7
  #include "duckdb/function/table/system_functions.hpp"
8
8
  #include "duckdb/main/database.hpp"
@@ -34,6 +34,7 @@ struct CreateIndexInfo;
34
34
  struct CreateTypeInfo;
35
35
  struct CreateTableInfo;
36
36
  struct DatabaseSize;
37
+ struct MetadataBlockInfo;
37
38
 
38
39
  class AttachedDatabase;
39
40
  class ClientContext;
@@ -266,6 +267,7 @@ public:
266
267
  unique_ptr<LogicalOperator> plan) = 0;
267
268
 
268
269
  virtual DatabaseSize GetDatabaseSize(ClientContext &context) = 0;
270
+ virtual vector<MetadataBlockInfo> GetMetadataInfo(ClientContext &context);
269
271
 
270
272
  virtual bool InMemory() = 0;
271
273
  virtual string GetDBPath() = 0;
@@ -54,6 +54,7 @@ public:
54
54
  unique_ptr<LogicalOperator> plan) override;
55
55
 
56
56
  DatabaseSize GetDatabaseSize(ClientContext &context) override;
57
+ vector<MetadataBlockInfo> GetMetadataInfo(ClientContext &context) override;
57
58
 
58
59
  DUCKDB_API bool InMemory() override;
59
60
  DUCKDB_API string GetDBPath() override;
@@ -15,10 +15,12 @@
15
15
  namespace duckdb {
16
16
  class ClientContext;
17
17
  class Catalog;
18
+ class DatabaseInstance;
18
19
  enum class ExpressionType : uint8_t;
19
20
 
20
21
  struct DeserializationData {
21
22
  stack<reference<ClientContext>> contexts;
23
+ stack<reference<DatabaseInstance>> databases;
22
24
  stack<idx_t> enums;
23
25
  stack<reference<bound_parameter_map_t>> parameter_data;
24
26
  stack<reference<LogicalType>> types;
@@ -74,6 +76,23 @@ inline void DeserializationData::Unset<LogicalOperatorType>() {
74
76
  enums.pop();
75
77
  }
76
78
 
79
+ template <>
80
+ inline void DeserializationData::Set(CompressionType type) {
81
+ enums.push(idx_t(type));
82
+ }
83
+
84
+ template <>
85
+ inline CompressionType DeserializationData::Get() {
86
+ AssertNotEmpty(enums);
87
+ return CompressionType(enums.top());
88
+ }
89
+
90
+ template <>
91
+ inline void DeserializationData::Unset<CompressionType>() {
92
+ AssertNotEmpty(enums);
93
+ enums.pop();
94
+ }
95
+
77
96
  template <>
78
97
  inline void DeserializationData::Set(CatalogType type) {
79
98
  enums.push(idx_t(type));
@@ -108,6 +127,23 @@ inline void DeserializationData::Unset<ClientContext>() {
108
127
  contexts.pop();
109
128
  }
110
129
 
130
+ template <>
131
+ inline void DeserializationData::Set(DatabaseInstance &db) {
132
+ databases.push(db);
133
+ }
134
+
135
+ template <>
136
+ inline DatabaseInstance &DeserializationData::Get() {
137
+ AssertNotEmpty(databases);
138
+ return databases.top();
139
+ }
140
+
141
+ template <>
142
+ inline void DeserializationData::Unset<DatabaseInstance>() {
143
+ AssertNotEmpty(databases);
144
+ databases.pop();
145
+ }
146
+
111
147
  template <>
112
148
  inline void DeserializationData::Set(bound_parameter_map_t &context) {
113
149
  parameter_data.push(context);