red-arrow-duckdb 1.0.1 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac2fc2780b82eadb1ac59bbd1af1427ef6522c4ce97c4a51c8d3dbfe7181763f
-  data.tar.gz: 4c75ea339bca32b4b3249faad455c7383ff700d7dd7d338804a91bd8ad4c46da
+  metadata.gz: 63fb3f7af8e84c386b1dcec7309b967d507ec85a65355471167c15426a5cf2f3
+  data.tar.gz: 9ae59ace06c47648b5fd4fa14723773de03f80c1f51f121e1b8fa3b225d1c56a
 SHA512:
-  metadata.gz: a1963496e56b1868cac0c462758250110c4a95d3bc1f990cb67f7e26da11db2f85ebe1a3edfcf3109454ff43a8b4012b9fb807164eb6fd089fe0ee176ef6718b
-  data.tar.gz: 637dd0226663c59358a13b6545544c49c149221d6e97b150d12751a50368e2c30dd8887e2b267728c10a0f228384fc17244ada2cf19cead1e8c51e5525ce32ca
+  metadata.gz: b63175c712cce70c9a48f2becb6581ff05ac5d153257bd740a178626222508f3f74c5bdab9e3618bc977e0b605fc79aa181c81cce8cfeaa33671d2ccf07bf763
+  data.tar.gz: 350c3eb4925cc5bd76af437c977f0cc8959c3ed91fcb4f119cb85c1c6f43a8709cb166dd993e53e9088edd4747b5950b875f0cc3548b03e512813c43b023e207

data/doc/text/news.md

@@ -1,5 +1,29 @@
 # News
 
+## 1.0.3 - 2024-10-11
+
+### Improvements
+
+  * Added support for DuckDB 1.1.0
+    * GH-4
+    * Patch by Sten Larsson
+
+### Thanks
+
+  * Sten Larsson
+
+## 1.0.2 - 2022-03-06
+
+### Improvements
+
+  * Added support for timestamp value pushdown.
+    [GitHub#2](https://github.com/red-data-tools/red-arrow-duckdb/issues/2)
+    [Reported by Torsten Sprenger]
+
+### Thanks
+
+  * Torsten Sprenger
+
 ## 1.0.1 - 2021-11-06
 
 ### Improvements
@@ -7,7 +31,8 @@
   * macOS: Don't require specifying `PKG_CONFIG_PATH` on install such
     as `PKG_CONFIG_PATH=$(brew --prefix openssl)/lib/pkgconfig gem
     install red-arrow-duckdb`.
-    [GitHub#1][Reported by Konstantin Ilchenko]
+    [GitHub#1](https://github.com/red-data-tools/red-arrow-duckdb/issues/2)
+    [Reported by Konstantin Ilchenko]
 
 ### Thanks
 

data/ext/arrow-duckdb/arrow-duckdb-registration.cpp

@@ -24,7 +24,8 @@
 #include <duckdb.hpp>
 #ifndef DUCKDB_AMALGAMATION
 #  include <duckdb.h>
-#  include <duckdb/common/arrow_wrapper.hpp>
+#  include <duckdb/common/arrow/arrow_wrapper.hpp>
+#  include <duckdb/function/table/arrow.hpp>
 #  include <duckdb/function/table_function.hpp>
 #  include <duckdb/main/connection.hpp>
 #  include <duckdb/planner/filter/conjunction_filter.hpp>
@@ -35,6 +36,22 @@
 #include "arrow-duckdb-registration.hpp"
 
 namespace {
+  std::shared_ptr<arrow::Scalar>
+  convert_constant_timestamp(duckdb::Value &value, arrow::TimeUnit::type unit)
+  {
+    auto scalar_result =
+      arrow::MakeScalar(arrow::timestamp(unit), value.GetValue<int64_t>());
+    if (!scalar_result.ok()) {
+      throw duckdb::InvalidInputException(
+        "[arrow][filter][pushdown][%s] "
+        "failed to convert to Apache Arrow scalar: %s: <%s>",
+        value.type().ToString(),
+        scalar_result.status().ToString(),
+        value.ToString());
+    }
+    return *scalar_result;
+  }
+
   std::shared_ptr<arrow::Scalar>
   convert_constant(duckdb::Value &value)
   {
@@ -55,9 +72,14 @@ namespace {
     //   return arrow::MakeScalar(arrow::date32(), value.GetValue<int32_t>());
     // case duckdb::LogicalTypeId::TIME:
     //   return arrow::MakeScalar(arrow::time64(), value.GetValue<int64_t>());
-    // case duckdb::LogicalTypeId::TIMESTAMP:
-    //   return arrow::MakeScalar(arrow::timestamp(),
-    //                            value.GetValue<int64_t>());
+    case duckdb::LogicalTypeId::TIMESTAMP_SEC:
+      return convert_constant_timestamp(value, arrow::TimeUnit::SECOND);
+    case duckdb::LogicalTypeId::TIMESTAMP_MS:
+      return convert_constant_timestamp(value, arrow::TimeUnit::MILLI);
+    case duckdb::LogicalTypeId::TIMESTAMP:
+      return convert_constant_timestamp(value, arrow::TimeUnit::MICRO);
+    case duckdb::LogicalTypeId::TIMESTAMP_NS:
+      return convert_constant_timestamp(value, arrow::TimeUnit::NANO);
     case duckdb::LogicalTypeId::UTINYINT:
       return arrow::MakeScalar(value.GetValue<uint8_t>());
     case duckdb::LogicalTypeId::USMALLINT:
@@ -75,7 +97,7 @@ namespace {
     // case LogicalTypeId::DECIMAL:
     default:
       throw duckdb::NotImplementedException(
-        "[arrow][filter][pushdown] not implemented value type: %s",
+        "[arrow][filter][pushdown][%s] not implemented value type",
         value.type().ToString());
     }
   }
@@ -141,14 +163,11 @@ namespace {
   }
 
   arrow::compute::Expression
-  convert_filters(std::unordered_map<
-                    idx_t,
-                    std::unique_ptr<duckdb::TableFilter>
-                  > &filters,
+  convert_filters(duckdb::TableFilterSet *filter_set,
                   std::unordered_map<idx_t, std::string> &column_names)
   {
     std::vector<arrow::compute::Expression> expressions;
-    for (auto it = filters.begin(); it != filters.end(); ++it) {
+    for (auto it = filter_set->filters.begin(); it != filter_set->filters.end(); ++it) {
       expressions.emplace_back(
         std::move(convert_filter(it->second.get(), column_names[it->first])));
     }
@@ -157,11 +176,7 @@ namespace {
 
   arrow::Result<std::unique_ptr<duckdb::ArrowArrayStreamWrapper>>
   arrow_table_produce_internal(uintptr_t data,
-                               std::pair<
-                                 std::unordered_map<idx_t, std::string>,
-                                 std::vector<std::string>
-                               > &project_columns,
-                               duckdb::TableFilterCollection *filters)
+                               duckdb::ArrowStreamParameters &parameters)
   {
     auto garrow_table = GARROW_TABLE(reinterpret_cast<gpointer>(data));
     auto arrow_table = garrow_table_get_raw(garrow_table);
@@ -169,20 +184,22 @@ namespace {
       std::make_shared<arrow::dataset::InMemoryDataset>(arrow_table);
     ARROW_ASSIGN_OR_RAISE(auto scanner_builder, dataset->NewScan());
     bool have_filter =
-      filters &&
-      filters->table_filters &&
-      !filters->table_filters->filters.empty();
+      parameters.filters &&
+      !parameters.filters->filters.empty();
     if (have_filter) {
       ARROW_RETURN_NOT_OK(
-        scanner_builder->Filter(convert_filters(filters->table_filters->filters,
-                                                project_columns.first)));
+        scanner_builder->Filter(
+          convert_filters(parameters.filters,
+                          parameters.projected_columns.projection_map)));
     }
-    if (!project_columns.second.empty()) {
-      ARROW_RETURN_NOT_OK(scanner_builder->Project(project_columns.second));
+    if (!parameters.projected_columns.columns.empty()) {
+      ARROW_RETURN_NOT_OK(
+        scanner_builder->Project(
+          parameters.projected_columns.columns));
     }
     ARROW_ASSIGN_OR_RAISE(auto scanner, scanner_builder->Finish());
     ARROW_ASSIGN_OR_RAISE(auto reader, scanner->ToRecordBatchReader());
-    auto stream_wrapper = duckdb::make_unique<duckdb::ArrowArrayStreamWrapper>();
+    auto stream_wrapper = duckdb::make_uniq<duckdb::ArrowArrayStreamWrapper>();
     ARROW_RETURN_NOT_OK(
       arrow::ExportRecordBatchReader(reader,
                                      &(stream_wrapper->arrow_array_stream)));
@@ -191,14 +208,9 @@ namespace {
 
   std::unique_ptr<duckdb::ArrowArrayStreamWrapper>
   arrow_table_produce(uintptr_t data,
-                      std::pair<
-                        std::unordered_map<idx_t, std::string>,
-                        std::vector<std::string>
-                      > &project_columns,
-                      duckdb::TableFilterCollection *filters)
+                      duckdb::ArrowStreamParameters &parameters)
   {
-    auto stream_wrapper_result =
-      arrow_table_produce_internal(data, project_columns, filters);
+    auto stream_wrapper_result = arrow_table_produce_internal(data, parameters);
     if (!stream_wrapper_result.ok()) {
       throw std::runtime_error(
         std::string("[arrow][produce] failed to produce: ") +
@@ -206,6 +218,20 @@ namespace {
     }
     return std::move(*stream_wrapper_result);
   }
+
+  void
+  arrow_table_get_schema(uintptr_t data, duckdb::ArrowSchemaWrapper &schema)
+  {
+    auto garrow_table = GARROW_TABLE(reinterpret_cast<gpointer>(data));
+    auto arrow_table = garrow_table_get_raw(garrow_table);
+    auto export_schema_status = arrow::ExportSchema(*(arrow_table->schema()),
+                                                    reinterpret_cast<ArrowSchema *>(&schema));
+    if (!export_schema_status.ok()) {
+      throw std::runtime_error(
+        std::string("[arrow][get_schema] failed to export schema: ") +
+        export_schema_status.ToString());
+    }
+  }
 }
 
 namespace arrow_duckdb {
@@ -224,14 +250,13 @@ namespace arrow_duckdb {
   {
     auto c_name = StringValueCStr(name);
     auto garrow_table = RVAL2GOBJ(arrow_table);
-    const idx_t rows_per_tuple = 1000000;
     reinterpret_cast<duckdb::Connection *>(connection)
       ->TableFunction(
         "arrow_scan",
         {
           duckdb::Value::POINTER(reinterpret_cast<uintptr_t>(garrow_table)),
           duckdb::Value::POINTER(reinterpret_cast<uintptr_t>(arrow_table_produce)),
-          duckdb::Value::UBIGINT(rows_per_tuple)
+          duckdb::Value::POINTER(reinterpret_cast<uintptr_t>(arrow_table_get_schema)),
         })
       ->CreateView(c_name, true, true);
   }

data/ext/arrow-duckdb/arrow-duckdb.cpp

@@ -20,7 +20,9 @@
 
 #include <rbgobject.h>
 
+extern "C" {
 #include <ruby-duckdb.h>
+}
 
 #include "arrow-duckdb-registration.hpp"
 
@@ -99,7 +101,7 @@ namespace {
   result_ensure_gschema(Result *result)
   {
     ArrowSchema c_abi_schema;
-    duckdb_arrow_schema schema = &c_abi_schema;
+    auto schema = reinterpret_cast<duckdb_arrow_schema>(&c_abi_schema);
     auto state = duckdb_query_arrow_schema(result->arrow, &schema);
     if (state == DuckDBError) {
       free(result->error_message);
@@ -121,7 +123,7 @@ namespace {
   result_fetch_internal(VALUE self, Result *result)
   {
     ArrowArray c_abi_array = {};
-    duckdb_arrow_array array = &c_abi_array;
+    auto array = reinterpret_cast<duckdb_arrow_array>(&c_abi_array);
     auto state = duckdb_query_arrow_array(result->arrow, &array);
     if (state == DuckDBError) {
       free(result->error_message);
@@ -219,8 +221,7 @@ namespace {
   VALUE
   query_sql_arrow(VALUE self, VALUE sql)
   {
-    rubyDuckDBConnection *ctx;
-    Data_Get_Struct(self, rubyDuckDBConnection, ctx);
+    auto ctx = get_struct_connection(self);
 
     if (!(ctx->con)) {
       rb_raise(eDuckDBError, "Database connection closed");
@@ -253,8 +254,7 @@ namespace {
   VALUE
   query_unregister_arrow(VALUE self, VALUE name)
   {
-    rubyDuckDBConnection *ctx;
-    Data_Get_Struct(self, rubyDuckDBConnection, ctx);
+    auto ctx = get_struct_connection(self);
 
     if (!(ctx->con)) {
       rb_raise(eDuckDBError, "Database connection closed");
@@ -293,8 +293,7 @@ namespace {
   VALUE
   query_register_arrow(VALUE self, VALUE name, VALUE arrow_table)
   {
-    rubyDuckDBConnection *ctx;
-    Data_Get_Struct(self, rubyDuckDBConnection, ctx);
+    auto ctx = get_struct_connection(self);
 
     if (!(ctx->con)) {
       rb_raise(eDuckDBError, "Database connection closed");
@@ -329,8 +328,7 @@ namespace {
   VALUE
   prepared_statement_execute_arrow(VALUE self)
   {
-    rubyDuckDBPreparedStatement *ctx;
-    Data_Get_Struct(self, rubyDuckDBPreparedStatement, ctx);
+    auto ctx = get_struct_prepared_statement(self);
 
     ID id_new;
     CONST_ID(id_new, "new");

data/ext/arrow-duckdb/extconf.rb

@@ -16,7 +16,7 @@ require "extpp"
 require "mkmf-gnome"
 require "native-package-installer"
 
-checking_for(checking_message("Homebrew")) do
+homebrew = checking_for(checking_message("Homebrew")) do
   case NativePackageInstaller::Platform.detect
   when NativePackageInstaller::Platform::Homebrew
     openssl_prefix = `brew --prefix openssl`.chomp
@@ -40,6 +40,10 @@ unless have_library("duckdb")
   install_missing_native_package(debian: "libduckdb-dev",
                                  redhat: "duckdb-devel",
                                  homebrew: "duckdb") or exit(false)
+  if homebrew
+    $INCFLAGS << " -I" << File.join(`brew --prefix duckdb`.chomp, "include")
+    $LIBPATH |= [File.join(`brew --prefix duckdb`.chomp, "lib")]
+  end
   have_library("duckdb") or exit(false)
 end
 

data/lib/arrow-duckdb/version.rb

@@ -1,4 +1,4 @@
-# Copyright 2021  Sutou Kouhei <kou@clear-code.com>
+# Copyright 2021-2022  Sutou Kouhei <kou@clear-code.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,5 +13,5 @@
 # limitations under the License.
 
 module ArrowDuckDB
-  VERSION = "1.0.1"
+  VERSION = "1.0.3"
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow-duckdb
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.3
 platform: ruby
 authors:
 - Sutou Kouhei
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-05 00:00:00.000000000 Z
+date: 2024-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: duckdb
@@ -104,7 +103,6 @@ homepage: https://github.com/red-data-tools/red-arrow-duckdb
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -119,8 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.0.dev
-signing_key: 
+rubygems_version: 3.6.0.dev
 specification_version: 4
 summary: Red Arrow DuckDB is a library that provides Apache Arrow support to ruby-duckdb.
 test_files: []