duckdb 0.9.1.2 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b492788be53be66a956ad49e4cd8a82ffa6273c12f037b2c0abd51253321802f
-  data.tar.gz: 50571029b1d1160a7abb6ac223bd4cda4f3791b29d26cdebac6b795f160a2a14
+  metadata.gz: afbb8627bfcb9764d6928d64558b55c18b26932b1ad50946cfe777169b6986f9
+  data.tar.gz: b560803df1d4b93d1e1b9f5c56b55f779c34095780592b9cdae51f7d4da2c0ec
 SHA512:
-  metadata.gz: d80355e181599217574191f3acba24d6dbcb37e4a73b774689590f42fc9b92032d4a4285962ee65100d3a3186825b29f09066d9fd0f8f24ce46aaec85cba4701
-  data.tar.gz: b126f7e0057c77cae338f7c6b76d4b775c52b06f84936ebd6208a3357148ababa8c8c3aaca727dacf4b7ed05b5f5da2bedc74d8b1d46f170ddc418d4c9e125fa
+  metadata.gz: 4c16acf784261f874501381aa9c764bc461d56fd35b066bae88c6cf0b4de654a867a6d309641cf0cf39d60854d268ce610fd3997534adcd11f322e7e4cd76401
+  data.tar.gz: 24d26d68b843c27ca306980e0d1acdfed2c6f24bb573cb89ba684879c4d05d43256d2335085e82fb1c352b7cdffcd016f2c2dadf3d924fc747cc4c450b142693

data/.gitattributes

@@ -0,0 +1 @@
+*.rb diff=ruby

data/.github/workflows/test_on_macos.yml

@@ -1,9 +1,9 @@
 name: MacOS
 
 on:
-  push:
-    branches:
-      - main
+  # push:
+  #   branches:
+  #     - main
   pull_request:
     types:
       - opened
@@ -15,8 +15,8 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        ruby: ['2.7.8', '3.0.6', '3.1.4', '3.2.2', '3.3.0-preview2', 'head']
-        duckdb: ['0.9.1', '0.8.1']
+        ruby: ['2.7.8', '3.0.6', '3.1.4', '3.2.2', '3.3.0-preview3', 'head']
+        duckdb: ['0.9.2', '0.8.1']
 
     steps:
     - uses: actions/checkout@v3
@@ -59,7 +59,7 @@ jobs:
 
     - name: run test with Ruby ${{ matrix.ruby }}
       run: |
-        rake test
+        env RUBYOPT=-W:deprecated rake test
 
   post-test:
     name: All tests passed on macos

data/.github/workflows/test_on_ubuntu.yml

@@ -1,9 +1,9 @@
 name: Ubuntu
 
 on:
-  push:
-    branches:
-      - main
+  # push:
+  #   branches:
+  #     - main
   pull_request:
     types:
       - opened
@@ -15,8 +15,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby: ['2.7.8', '3.0.6', '3.1.4', '3.2.2', '3.3.0-preview2', 'head']
-        duckdb: ['0.9.1', '0.8.1']
+        ruby: ['2.7.8', '3.0.6', '3.1.4', '3.2.2', '3.3.0-preview3', 'head']
+        duckdb: ['0.9.2', '0.8.1']
 
     steps:
     - uses: actions/checkout@v3
@@ -57,7 +57,7 @@ jobs:
       env:
         DUCKDB_VERSION: ${{ matrix.duckdb }}
       run: |
-        rake test
+        env RUBYOPT=-W:deprecated rake test
 
   post-test:
     name: All tests passed on Ubuntu

data/.github/workflows/test_on_windows.yml

@@ -1,9 +1,9 @@
 name: Windows
 
 on:
-  push:
-    branches:
-      - main
+  # push:
+  #   branches:
+  #     - main
   pull_request:
     types:
       - opened
@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         ruby: ['2.7.8', '3.0.6', '3.1.4', '3.2.2', 'ucrt', 'mingw', 'mswin', 'head']
-        duckdb: ['0.9.1', '0.8.1']
+        duckdb: ['0.9.2', '0.8.1']
 
     steps:
     - uses: actions/checkout@v3

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # ChangeLog
 
+# 0.9.2
+- add DuckDB::Connection#async_query_stream
+- DuckDB::PendingResult accepts second argument. If the second argument is
+  true, PendingResult#execute_pending returns streaming DuckDB::Result object.
+- add DuckDB::PreparedStatement#pending_prepared_stream
+- add DuckDB::Result#streaming?.
+
 # 0.9.1.2
 - add DuckDB::Connection#interrupt, DuckDB::Connection#query_progress
 - add DuckDB::Connection#async_query, alias method async_execute.

data/Dockerfile

@@ -1,7 +1,7 @@
 ARG RUBY_VERSION=3.2.2
 FROM ruby:${RUBY_VERSION}
 
-ARG DUCKDB_VERSION=0.9.1
+ARG DUCKDB_VERSION=0.9.2
 
 RUN apt update -qq && \
     apt install -y build-essential curl git wget

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    duckdb (0.9.1.2)
+    duckdb (0.9.2)
 
 GEM
   remote: https://rubygems.org/
@@ -9,16 +9,16 @@ GEM
     benchmark-ips (2.12.0)
     mini_portile2 (2.8.5)
     minitest (5.20.0)
-    nokogiri (1.15.4)
+    nokogiri (1.15.5)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    nokogiri (1.15.4-x86_64-linux)
+    nokogiri (1.15.5-x86_64-linux)
       racc (~> 1.4)
     racc (1.7.3)
     rake (13.1.0)
     rake-compiler (1.2.5)
       rake
-    ruby_memcheck (2.2.0)
+    ruby_memcheck (2.2.1)
       nokogiri
     stackprof (0.2.25)
 

data/README.md

@@ -104,10 +104,25 @@ con.query('SELECT * FROM users WHERE name = ? AND email = ?', 'Alice', 'alice@ex
 con.query('SELECT * FROM users WHERE name = $name AND email = $email', name: 'Alice', email: 'alice@example.com')
 ```
 
+### using async query
+
+You can use async query.
+
+```ruby
+DuckDB::Result.use_chunk_each = true # must be true.
+...
+
+pending_result = con.async_query_stream('SLOW QUERY')
+pending_result.execute_task while pending_result.state == :not_ready
+
+result = pending_result.execute_pending
+result.each.first
+```
+
+Here is [the benchmark](./benchmark/async_query.rb).
 
 ### using BLOB column
 
-BLOB is available with DuckDB v0.2.5 or later.
 Use `DuckDB::Blob.new` or use sting#force_encoding(Encoding::BINARY)
 
 ```ruby
@@ -119,6 +134,7 @@ DuckDB::Database.open do |db|
     stmt = DuckDB::PreparedStatement.new(con, 'INSERT INTO blob_table VALUES ($1)')
 
     stmt.bind(1, DuckDB::Blob.new("\0\1\2\3\4\5"))
+    #  or
     # stmt.bind(1, "\0\1\2\3\4\5".force_encoding(Encoding::BINARY))
     stmt.execute
 

data/benchmark/async_query.rb

@@ -0,0 +1,90 @@
+require 'bundler/setup'
+require 'duckdb'
+require 'benchmark/ips'
+
+
+DuckDB::Result.use_chunk_each = true
+DuckDB::Database.open do |db|
+  db.connect do |con|
+    con.query('SET threads=1')
+    con.query('CREATE TABLE tbl as SELECT range a, mod(range, 10) b FROM range(100000)')
+    con.query('CREATE TABLE tbl2 as SELECT range a, mod(range, 10) b FROM range(100000)')
+    query_sql = 'SELECT * FROM tbl where b = (SELECT min(b) FROM tbl2)'
+    print <<~END_OF_HEAD
+
+      Benchmark: Get first record ======================================
+    END_OF_HEAD
+
+    Benchmark.ips do |x|
+      x.report('async_query') do
+        pending_result = con.async_query(query_sql)
+
+        pending_result.execute_task while pending_result.state == :not_ready
+        result = pending_result.execute_pending
+        result.each.first
+      end
+      x.report('query') do
+        result = con.query(query_sql)
+        result.each.first
+      end
+      x.report('async_query_stream') do
+        pending_result = con.async_query_stream(query_sql)
+
+        pending_result.execute_task while pending_result.state == :not_ready
+        result = pending_result.execute_pending
+        result.each.first
+      end
+    end
+
+    print <<~END_OF_HEAD
+
+
+      Benchmark: Get all records ======================================
+    END_OF_HEAD
+
+    Benchmark.ips do |x|
+      x.report('async_query') do
+        pending_result = con.async_query(query_sql)
+
+        pending_result.execute_task while pending_result.state == :not_ready
+        result = pending_result.execute_pending
+        result.each.to_a
+      end
+      x.report('query') do
+        result = con.query(query_sql)
+        result.each.to_a
+      end
+      x.report('async_query_stream') do
+        pending_result = con.async_query_stream(query_sql)
+
+        pending_result.execute_task while pending_result.state == :not_ready
+        result = pending_result.execute_pending
+        result.each.to_a
+      end
+    end
+  end
+end
+
+__END__
+
+results:
+Benchmark: Get first record ======================================
+Warming up --------------------------------------
+         async_query    70.000  i/100ms
+               query    88.000  i/100ms
+  async_query_stream   188.000  i/100ms
+Calculating -------------------------------------
+         async_query    847.191  (± 4.6%) i/s -      4.270k in   5.051650s
+               query    850.509  (± 3.8%) i/s -      4.312k in   5.078167s
+  async_query_stream      1.757k (± 7.3%) i/s -      8.836k in   5.057142s
+
+
+Benchmark: Get all records ======================================
+Warming up --------------------------------------
+         async_query    40.000  i/100ms
+               query    40.000  i/100ms
+  async_query_stream    39.000  i/100ms
+Calculating -------------------------------------
+         async_query    402.567  (± 0.5%) i/s -      2.040k in   5.067639s
+               query    406.632  (± 0.7%) i/s -      2.040k in   5.017079s
+  async_query_stream    395.532  (± 0.8%) i/s -      1.989k in   5.028955s

data/ext/duckdb/extconf.rb

@@ -1,31 +1,64 @@
+# frozen_string_literal: true
+
 require 'mkmf'
 
-def duckdb_library_available?(func)
-  header = find_header('duckdb.h') || find_header('duckdb.h', '/opt/homebrew/include')
-  library = have_func(func, 'duckdb.h') || find_library('duckdb', func, '/opt/homebrew/opt/duckdb/lib')
-  header && library
+DUCKDB_REQUIRED_VERSION = '0.8.0'
+
+def check_duckdb_header(header, version)
+  found = find_header(
+    header,
+    '/opt/homebrew/include',
+    '/opt/homebrew/opt/duckdb/include',
+    '/opt/local/include'
+  )
+  return if found
+
+  msg = "#{header} is not found. Install #{header} of duckdb >= #{version}."
+  print_message(msg)
+  raise msg
 end
 
-def check_duckdb_library(func, version)
-  return if duckdb_library_available?(func)
+def check_duckdb_library(library, func, version)
+  found = find_library(
+    library,
+    func,
+    '/opt/homebrew/lib',
+    '/opt/homebrew/opt/duckdb/lib',
+    '/opt/local/lib'
+  )
+  return if found
 
-  msg = "duckdb >= #{version} is not found. Install duckdb >= #{version} library and header file."
-  puts ''
-  puts '*' * 80
-  puts msg
-  puts '*' * 80
-  puts ''
+  library_name = duckdb_library_name(library)
+  msg = "#{library_name} is not found. Install #{library_name} of duckdb >= #{version}."
+  print_message(msg)
   raise msg
 end
 
+def duckdb_library_name(library)
+  "lib#{library}.(so|dylib|dll)"
+end
+
+def print_message(msg)
+  print <<~END_OF_MESSAGE
+
+    #{'*' * 80}
+    #{msg}
+    #{'*' * 80}
+
+  END_OF_MESSAGE
+end
+
 dir_config('duckdb')
 
+check_duckdb_header('duckdb.h', DUCKDB_REQUIRED_VERSION)
+
 # check duckdb >= 0.8.0
-check_duckdb_library('duckdb_string_is_inlined', '0.8.0')
+check_duckdb_library('duckdb', 'duckdb_string_is_inlined', DUCKDB_REQUIRED_VERSION)
 
 # check duckdb >= 0.9.0
 have_func('duckdb_bind_parameter_index', 'duckdb.h')
 
+# duckdb_parameter_name is not found on Windows.
 have_func('duckdb_parameter_name', 'duckdb.h')
 
 create_makefile('duckdb/duckdb_native')

data/ext/duckdb/pending_result.c

@@ -5,7 +5,7 @@ static VALUE cDuckDBPendingResult;
 static void deallocate(void *ctx);
 static VALUE allocate(VALUE klass);
 static size_t memsize(const void *p);
-static VALUE duckdb_pending_result_initialize(VALUE self, VALUE oDuckDBPreparedStatement);
+static VALUE duckdb_pending_result_initialize(int argc, VALUE *args, VALUE self);
 static VALUE duckdb_pending_result_execute_task(VALUE self);
 static VALUE duckdb_pending_result_execute_pending(VALUE self);
 
@@ -38,11 +38,27 @@ static size_t memsize(const void *p) {
     return sizeof(rubyDuckDBPendingResult);
 }
 
-static VALUE duckdb_pending_result_initialize(VALUE self, VALUE oDuckDBPreparedStatement) {
+static VALUE duckdb_pending_result_initialize(int argc, VALUE *argv, VALUE self) {
+    VALUE oDuckDBPreparedStatement;
+    VALUE streaming_p = Qfalse;
+    duckdb_state state;
+
+    rb_scan_args(argc, argv, "11", &oDuckDBPreparedStatement, &streaming_p);
+
+    if (rb_obj_is_kind_of(oDuckDBPreparedStatement, cDuckDBPreparedStatement) != Qtrue) {
+        rb_raise(rb_eTypeError, "1st argument must be DuckDB::PreparedStatement");
+    }
+
     rubyDuckDBPendingResult *ctx = get_struct_pending_result(self);
     rubyDuckDBPreparedStatement *stmt = get_struct_prepared_statement(oDuckDBPreparedStatement);
 
-    if (duckdb_pending_prepared(stmt->prepared_statement, &(ctx->pending_result)) == DuckDBError) {
+    if (!NIL_P(streaming_p) && streaming_p == Qtrue) {
+        state = duckdb_pending_prepared_streaming(stmt->prepared_statement, &(ctx->pending_result));
+    } else {
+        state = duckdb_pending_prepared(stmt->prepared_statement, &(ctx->pending_result));
+    }
+
+    if (state == DuckDBError) {
         rb_raise(eDuckDBError, "%s", duckdb_pending_error(ctx->pending_result));
     }
     return self;
@@ -112,7 +128,7 @@ void rbduckdb_init_duckdb_pending_result(void) {
     cDuckDBPendingResult = rb_define_class_under(mDuckDB, "PendingResult", rb_cObject);
     rb_define_alloc_func(cDuckDBPendingResult, allocate);
 
-    rb_define_method(cDuckDBPendingResult, "initialize", duckdb_pending_result_initialize, 1);
+    rb_define_method(cDuckDBPendingResult, "initialize", duckdb_pending_result_initialize, -1);
     rb_define_method(cDuckDBPendingResult, "execute_task", duckdb_pending_result_execute_task, 0);
     rb_define_method(cDuckDBPendingResult, "execute_pending", duckdb_pending_result_execute_pending, 0);
 

data/ext/duckdb/prepared_statement.c

@@ -1,6 +1,6 @@
 #include "ruby-duckdb.h"
 
-static VALUE cDuckDBPreparedStatement;
+VALUE cDuckDBPreparedStatement;
 
 static void deallocate(void *ctx);
 static VALUE allocate(VALUE klass);

data/ext/duckdb/result.c

@@ -19,6 +19,11 @@ static VALUE duckdb_result_column_count(VALUE oDuckDBResult);
 static VALUE duckdb_result_row_count(VALUE oDuckDBResult);
 static VALUE duckdb_result_rows_changed(VALUE oDuckDBResult);
 static VALUE duckdb_result_columns(VALUE oDuckDBResult);
+static VALUE duckdb_result_streaming_p(VALUE oDuckDBResult);
+static VALUE duckdb_result_chunk_each(VALUE oDuckDBResult);
+
+static VALUE duckdb_result__chunk_stream(VALUE oDuckDBResult);
+static void yield_rows(duckdb_data_chunk chunk, idx_t col_count);
 static VALUE duckdb_result__column_type(VALUE oDuckDBResult, VALUE col_idx);
 static VALUE duckdb_result__is_null(VALUE oDuckDBResult, VALUE row_idx, VALUE col_idx);
 static VALUE duckdb_result__to_boolean(VALUE oDuckDBResult, VALUE row_idx, VALUE col_idx);
@@ -50,7 +55,6 @@ static VALUE vector_map(duckdb_logical_type ty, duckdb_vector vector, idx_t row_
 static VALUE vector_struct(duckdb_logical_type ty, duckdb_vector vector, idx_t row_idx);
 static VALUE vector_uuid(void* vector_data, idx_t row_idx);
 static VALUE vector_value(duckdb_vector vector, idx_t row_idx);
-static VALUE duckdb_result_chunk_each(VALUE oDuckDBResult);
 
 static const rb_data_type_t result_data_type = {
     "DuckDB/Result",
@@ -235,6 +239,79 @@ static VALUE duckdb_result_columns(VALUE oDuckDBResult) {
     return ary;
 }
 
+/*
+ *  call-seq:
+ *    result.streaming? -> Boolean
+ *
+ *  Returns true if the result is streaming, otherwise false.
+ *
+ */
+static VALUE duckdb_result_streaming_p(VALUE oDuckDBResult) {
+    rubyDuckDBResult *ctx;
+    TypedData_Get_Struct(oDuckDBResult, rubyDuckDBResult, &result_data_type, ctx);
+    return duckdb_result_is_streaming(ctx->result) ? Qtrue : Qfalse;
+}
+
+static VALUE duckdb_result_chunk_each(VALUE oDuckDBResult) {
+    rubyDuckDBResult *ctx;
+    idx_t col_count;
+    idx_t chunk_count;
+    idx_t chunk_idx;
+    duckdb_data_chunk chunk;
+
+    TypedData_Get_Struct(oDuckDBResult, rubyDuckDBResult, &result_data_type, ctx);
+
+    col_count = duckdb_column_count(&(ctx->result));
+    chunk_count = duckdb_result_chunk_count(ctx->result);
+
+    RETURN_ENUMERATOR(oDuckDBResult, 0, 0);
+
+    for (chunk_idx = 0; chunk_idx < chunk_count; chunk_idx++) {
+        chunk = duckdb_result_get_chunk(ctx->result, chunk_idx);
+        yield_rows(chunk, col_count);
+        duckdb_destroy_data_chunk(&chunk);
+    }
+    return Qnil;
+}
+
+static VALUE duckdb_result__chunk_stream(VALUE oDuckDBResult) {
+    rubyDuckDBResult *ctx;
+    duckdb_data_chunk chunk;
+    idx_t col_count;
+
+    TypedData_Get_Struct(oDuckDBResult, rubyDuckDBResult, &result_data_type, ctx);
+
+    RETURN_ENUMERATOR(oDuckDBResult, 0, 0);
+
+    col_count = duckdb_column_count(&(ctx->result));
+
+    while((chunk = duckdb_stream_fetch_chunk(ctx->result)) != NULL) {
+        yield_rows(chunk, col_count);
+        duckdb_destroy_data_chunk(&chunk);
+    }
+    return Qnil;
+}
+
+static void yield_rows(duckdb_data_chunk chunk, idx_t col_count) {
+    idx_t row_count;
+    idx_t row_idx;
+    idx_t col_idx;
+    duckdb_vector vector;
+    VALUE row;
+    VALUE val;
+
+    row_count = duckdb_data_chunk_get_size(chunk);
+    for (row_idx = 0; row_idx < row_count; row_idx++) {
+        row = rb_ary_new2(col_count);
+        for (col_idx = 0; col_idx < col_count; col_idx++) {
+            vector = duckdb_data_chunk_get_vector(chunk, col_idx);
+            val = vector_value(vector, row_idx);
+            rb_ary_store(row, col_idx, val);
+        }
+        rb_yield(row);
+    }
+}
+
 static VALUE duckdb_result__column_type(VALUE oDuckDBResult, VALUE col_idx) {
     rubyDuckDBResult *ctx;
     TypedData_Get_Struct(oDuckDBResult, rubyDuckDBResult, &result_data_type, ctx);
@@ -405,32 +482,32 @@ static VALUE vector_date(void *vector_data, idx_t row_idx) {
     duckdb_date_struct date = duckdb_from_date(((duckdb_date *) vector_data)[row_idx]);
 
     return rb_funcall(mDuckDBConverter, rb_intern("_to_date"), 3,
-            INT2FIX(date.year),
-            INT2FIX(date.month),
-            INT2FIX(date.day)
-            );
+                      INT2FIX(date.year),
+                      INT2FIX(date.month),
+                      INT2FIX(date.day)
+                      );
 }
 
 static VALUE vector_timestamp(void* vector_data, idx_t row_idx) {
     duckdb_timestamp_struct data = duckdb_from_timestamp(((duckdb_timestamp *)vector_data)[row_idx]);
     return rb_funcall(mDuckDBConverter, rb_intern("_to_time"), 7,
-            INT2FIX(data.date.year),
-            INT2FIX(data.date.month),
-            INT2FIX(data.date.day),
-            INT2FIX(data.time.hour),
-            INT2FIX(data.time.min),
-            INT2FIX(data.time.sec),
-            INT2NUM(data.time.micros)
-            );
+                      INT2FIX(data.date.year),
+                      INT2FIX(data.date.month),
+                      INT2FIX(data.date.day),
+                      INT2FIX(data.time.hour),
+                      INT2FIX(data.time.min),
+                      INT2FIX(data.time.sec),
+                      INT2NUM(data.time.micros)
+                      );
 }
 
 static VALUE vector_interval(void* vector_data, idx_t row_idx) {
     duckdb_interval data = ((duckdb_interval *)vector_data)[row_idx];
     return rb_funcall(mDuckDBConverter, rb_intern("_to_interval_from_vector"), 3,
-            INT2NUM(data.months),
-            INT2NUM(data.days),
-            LL2NUM(data.micros)
-            );
+                      INT2NUM(data.months),
+                      INT2NUM(data.days),
+                      LL2NUM(data.micros)
+                      );
 }
 
 static VALUE vector_blob(void* vector_data, idx_t row_idx) {
@@ -454,9 +531,9 @@ static VALUE vector_varchar(void* vector_data, idx_t row_idx) {
 static VALUE vector_hugeint(void* vector_data, idx_t row_idx) {
     duckdb_hugeint hugeint = ((duckdb_hugeint *)vector_data)[row_idx];
     return rb_funcall(mDuckDBConverter, rb_intern("_to_hugeint_from_vector"), 2,
-            ULL2NUM(hugeint.lower),
-            LL2NUM(hugeint.upper)
-            );
+                      ULL2NUM(hugeint.lower),
+                      LL2NUM(hugeint.upper)
+                      );
 }
 
 static VALUE vector_decimal(duckdb_logical_type ty, void* vector_data, idx_t row_idx) {
@@ -477,11 +554,11 @@ static VALUE vector_decimal(duckdb_logical_type ty, void* vector_data, idx_t row
     }
 
     return rb_funcall(mDuckDBConverter, rb_intern("_to_decimal_from_vector"), 4,
-            INT2FIX(width),
-            INT2FIX(scale),
-            ULL2NUM(value.lower),
-            LL2NUM(value.upper)
-            );
+                      INT2FIX(width),
+                      INT2FIX(scale),
+                      ULL2NUM(value.lower),
+                      LL2NUM(value.upper)
+                      );
 }
 
 static VALUE vector_enum(duckdb_logical_type ty, void* vector_data, idx_t row_idx) {
@@ -577,9 +654,9 @@ static VALUE vector_struct(duckdb_logical_type ty, duckdb_vector vector, idx_t r
 static VALUE vector_uuid(void* vector_data, idx_t row_idx) {
     duckdb_hugeint hugeint = ((duckdb_hugeint *)vector_data)[row_idx];
     return rb_funcall(mDuckDBConverter, rb_intern("_to_uuid_from_vector"), 2,
-            ULL2NUM(hugeint.lower),
-            LL2NUM(hugeint.upper)
-            );
+                      ULL2NUM(hugeint.lower),
+                      LL2NUM(hugeint.upper)
+                      );
 }
 
 static VALUE vector_value(duckdb_vector vector, idx_t row_idx) {
@@ -617,7 +694,7 @@ static VALUE vector_value(duckdb_vector vector, idx_t row_idx) {
         case DUCKDB_TYPE_BIGINT:
             obj = LL2NUM(((int64_t *) vector_data)[row_idx]);
             break;
-	    case DUCKDB_TYPE_UTINYINT:
+        case DUCKDB_TYPE_UTINYINT:
             obj = INT2FIX(((uint8_t *) vector_data)[row_idx]);
             break;
         case DUCKDB_TYPE_USMALLINT:
@@ -680,43 +757,6 @@ static VALUE vector_value(duckdb_vector vector, idx_t row_idx) {
     return obj;
 }
 
-static VALUE duckdb_result_chunk_each(VALUE oDuckDBResult) {
-    rubyDuckDBResult *ctx;
-    VALUE row;
-    idx_t col_count;
-    idx_t row_count;
-    idx_t chunk_count;
-    idx_t col_idx;
-    idx_t row_idx;
-    idx_t chunk_idx;
-    duckdb_data_chunk chunk;
-    duckdb_vector vector;
-    VALUE val;
-
-    TypedData_Get_Struct(oDuckDBResult, rubyDuckDBResult, &result_data_type, ctx);
-
-    col_count = duckdb_column_count(&(ctx->result));
-    chunk_count = duckdb_result_chunk_count(ctx->result);
-
-    RETURN_ENUMERATOR(oDuckDBResult, 0, 0);
-
-    for (chunk_idx = 0; chunk_idx < chunk_count; chunk_idx++) {
-        chunk = duckdb_result_get_chunk(ctx->result, chunk_idx);
-        row_count = duckdb_data_chunk_get_size(chunk);
-        for (row_idx = 0; row_idx < row_count; row_idx++) {
-            row = rb_ary_new2(col_count);
-            for (col_idx = 0; col_idx < col_count; col_idx++) {
-                vector = duckdb_data_chunk_get_vector(chunk, col_idx);
-                val = vector_value(vector, row_idx);
-                rb_ary_store(row, col_idx, val);
-            }
-            rb_yield(row);
-        }
-        duckdb_destroy_data_chunk(&chunk);
-    }
-    return Qnil;
-}
-
 void rbduckdb_init_duckdb_result(void) {
     cDuckDBResult = rb_define_class_under(mDuckDB, "Result", rb_cObject);
     rb_define_alloc_func(cDuckDBResult, allocate);
@@ -725,6 +765,9 @@ void rbduckdb_init_duckdb_result(void) {
     rb_define_method(cDuckDBResult, "row_count", duckdb_result_row_count, 0);
     rb_define_method(cDuckDBResult, "rows_changed", duckdb_result_rows_changed, 0);
     rb_define_method(cDuckDBResult, "columns", duckdb_result_columns, 0);
+    rb_define_method(cDuckDBResult, "streaming?", duckdb_result_streaming_p, 0);
+    rb_define_method(cDuckDBResult, "chunk_each", duckdb_result_chunk_each, 0);
+    rb_define_private_method(cDuckDBResult, "_chunk_stream", duckdb_result__chunk_stream, 0);
     rb_define_private_method(cDuckDBResult, "_column_type", duckdb_result__column_type, 1);
     rb_define_private_method(cDuckDBResult, "_null?", duckdb_result__is_null, 2);
     rb_define_private_method(cDuckDBResult, "_to_boolean", duckdb_result__to_boolean, 2);
@@ -742,5 +785,4 @@ void rbduckdb_init_duckdb_result(void) {
     rb_define_private_method(cDuckDBResult, "_enum_internal_type", duckdb_result__enum_internal_type, 1);
     rb_define_private_method(cDuckDBResult, "_enum_dictionary_size", duckdb_result__enum_dictionary_size, 1);
     rb_define_private_method(cDuckDBResult, "_enum_dictionary_value", duckdb_result__enum_dictionary_value, 2);
-    rb_define_method(cDuckDBResult, "chunk_each", duckdb_result_chunk_each, 0);
 }

data/ext/duckdb/ruby-duckdb.h

@@ -29,5 +29,6 @@ extern VALUE cDuckDBBlob;
 extern VALUE cDuckDBConfig;
 extern VALUE eDuckDBError;
 extern VALUE mDuckDBConverter;
+extern VALUE cDuckDBPreparedStatement;
 
 #endif

data/lib/duckdb/connection.rb

@@ -42,14 +42,12 @@ module DuckDB
     #   require 'duckdb'
     #   db = DuckDB::Database.open('duckdb_file')
     #   con = db.connect
-    #   pending_result = con.async_query('SELECT * FROM users')
-    #   sql = 'SELECT * FROM users WHERE name = ? AND email = ?'
-    #   pending_result = con.async_query(sql, 'Dave', 'dave@example.com')
-    #
-    #   # or You can use named parameter.
     #
     #   sql = 'SELECT * FROM users WHERE name = $name AND email = $email'
     #   pending_result = con.async_query(sql, name: 'Dave', email: 'dave@example.com')
+    #   pending_result.execute_task while pending_result.state == :not_ready
+    #   result = pending_result.execute_pending
+    #   result.each.first
     #
     def async_query(sql, *args, **kwargs)
       stmt = PreparedStatement.new(self, sql)
@@ -57,6 +55,30 @@ module DuckDB
       stmt.pending_prepared
     end
 
+    #
+    # executes sql with args asynchronously and provides streaming result.
+    # The first argument sql must be SQL string.
+    # The rest arguments are parameters of SQL string.
+    # This method returns DuckDB::PendingResult object.
+    #
+    #   require 'duckdb'
+    #   DuckDB::Result.use_chunk_each = true # must be true
+    #   db = DuckDB::Database.open('duckdb_file')
+    #   con = db.connect
+    #
+    #   sql = 'SELECT * FROM users WHERE name = $name AND email = $email'
+    #   pending_result = con.async_query_stream(sql, name: 'Dave', email: 'dave@example.com')
+    #
+    #   pending_result.execute_task while pending_result.state == :not_ready
+    #   result = pending_result.execute_pending
+    #   result.each.first
+    #
+    def async_query_stream(sql, *args, **kwargs)
+      stmt = PreparedStatement.new(self, sql)
+      stmt.bind_args(*args, **kwargs)
+      stmt.pending_prepared_stream
+    end
+
     #
     # connects DuckDB database
     # The first argument is DuckDB::Database object

data/lib/duckdb/prepared_statement.rb

@@ -24,6 +24,12 @@ module DuckDB
       PendingResult.new(self)
     end
 
+    def pending_prepared_stream
+      raise DuckDB::Error, 'DuckDB::Result.use_chunk_each must be true.' unless DuckDB::Result.use_chunk_each?
+
+      PendingResult.new(self, true)
+    end
+
     # binds all parameters with SQL prepared statement.
     #
     #   require 'duckdb'

data/lib/duckdb/result.rb

@@ -58,9 +58,15 @@ module DuckDB
 
     def each
       if self.class.use_chunk_each?
-        return chunk_each unless block_given?
+        if streaming?
+          return _chunk_stream unless block_given?
 
-        chunk_each { |row| yield row }
+          _chunk_stream { |row| yield row }
+        else
+          return chunk_each unless block_given?
+
+          chunk_each { |row| yield row }
+        end
       else
         warn('this `each` behavior will be deprecated in the future. set `DuckDB::Result.use_chunk_each = true` to use new `each` behavior.')
         return to_enum { row_size } unless block_given?

data/lib/duckdb/version.rb

@@ -3,5 +3,5 @@
 module DuckDB
   # The version string of ruby-duckdb.
   # Currently, ruby-duckdb is NOT semantic versioning.
-  VERSION = '0.9.1.2'
+  VERSION = '0.9.2'
 end

data/sample/async_query.rb

@@ -0,0 +1,24 @@
+require 'duckdb'
+
+DuckDB::Result.use_chunk_each = true
+DuckDB::Database.open do |db|
+  db.connect do |con|
+    con.query('SET threads=1')
+    con.query('CREATE TABLE tbl as SELECT range a, mod(range, 10) b FROM range(10000)')
+    con.query('CREATE TABLE tbl2 as SELECT range a, mod(range, 10) b FROM range(10000)')
+    # con.query('SET ENABLE_PROGRESS_BAR=true')
+    # con.query('SET ENABLE_PROGRESS_BAR_PRINT=false')
+    pending_result = con.async_query('SELECT * FROM tbl where b = (SELECT min(b) FROM tbl2)')
+
+    # con.interrupt
+    while pending_result.state == :not_ready
+      pending_result.execute_task
+      print '.'
+      $stdout.flush
+      sleep 0.01
+    end
+    result = pending_result.execute_pending
+    puts
+    p result.each.first
+  end
+end

data/sample/async_query_stream.rb

@@ -0,0 +1,24 @@
+require 'duckdb'
+
+DuckDB::Result.use_chunk_each = true
+DuckDB::Database.open do |db|
+  db.connect do |con|
+    con.query('SET threads=1')
+    con.query('CREATE TABLE tbl as SELECT range a, mod(range, 10) b FROM range(10000)')
+    con.query('CREATE TABLE tbl2 as SELECT range a, mod(range, 10) b FROM range(10000)')
+    # con.query('SET ENABLE_PROGRESS_BAR=true')
+    # con.query('SET ENABLE_PROGRESS_BAR_PRINT=false')
+    pending_result = con.async_query_stream('SELECT * FROM tbl where b = (SELECT min(b) FROM tbl2)')
+
+    # con.interrupt
+    while pending_result.state == :not_ready
+      pending_result.execute_task
+      print '.'
+      $stdout.flush
+      sleep 0.01
+    end
+    result = pending_result.execute_pending
+    puts
+    p result.each.first
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: duckdb
 version: !ruby/object:Gem::Version
-  version: 0.9.1.2
+  version: 0.9.2
 platform: ruby
 authors:
 - Masaki Suketa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-11-05 00:00:00.000000000 Z
+date: 2023-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -75,6 +75,7 @@ extensions:
 - ext/duckdb/extconf.rb
 extra_rdoc_files: []
 files:
+- ".gitattributes"
 - ".github/FUNDING.yml"
 - ".github/workflows/test_on_macos.yml"
 - ".github/workflows/test_on_ubuntu.yml"
@@ -88,6 +89,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- benchmark/async_query.rb
 - benchmark/converter_hugeint_ips.rb
 - benchmark/get_converter_module_ips.rb
 - benchmark/to_bigdecimal_ips.rb
@@ -138,6 +140,8 @@ files:
 - lib/duckdb/prepared_statement.rb
 - lib/duckdb/result.rb
 - lib/duckdb/version.rb
+- sample/async_query.rb
+- sample/async_query_stream.rb
 homepage: https://github.com/suketa/ruby-duckdb
 licenses:
 - MIT