extralite 2.4 → 2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8ccbfbf37744a5b90d30038c3ba4e73f7d2a094a0f18f21cd8b58d66ec32b42
-  data.tar.gz: 5c6d3f9c967f7056f8eeb7ba471e2db50a26be334bdeec7ddf80dbd521efff77
+  metadata.gz: dd2ac51ceb97cc6476f169da43fd11d6dad00ac39d5e9fd5eb0d4842ce670f0e
+  data.tar.gz: f58478324015a1f1827c55ca8b2170b177b270cad0ab8102be225a414da34148
 SHA512:
-  metadata.gz: 19f10ce33c25e3b0837ec452b8eff0c4c433a729c79162d8b73e65411142f6df61703e0d7c254ac269fbf02ae39aa1c0cde5cb8f9fe3f262ab9ad0af503ceb99
-  data.tar.gz: 921aeeb63e23bfc145af2c978fb3ce5a47b643cf23e265aaf581eb067e8a76ef5c63effde30c567d7565a19fa8f844868456f996d5413a70ef2ba774a927d638
+  metadata.gz: 3051d9b5d2a2543be9bdafd5b46aab5574acefa8ca946eaf34f5b1a8097154748f90ab6ba1b7fd76d5bb2b92d97599b35748be0d815258b4b3a499c6c5cefef7
+  data.tar.gz: 148d29ccfff66b0f1531000d0773b8de98b98169594c08ed032fccd7e92ffe1333357aa3da2e168d2b629d4c7b41b14af373e705601bf384b9d4a4f39af9c4f6

data/.github/workflows/test-bundle.yml

@@ -0,0 +1,30 @@
+name: Tests (extralite-bundle)
+
+on: [push, pull_request]
+
+concurrency:
+  group: tests-${{ format('{0}-{1}', github.head_ref || github.run_number, github.job) }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        ruby: ['3.0', '3.1', '3.2', '3.3']
+
+    name: >-
+      ${{matrix.os}}, ${{matrix.ruby}}
+
+    runs-on: ${{matrix.os}}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{matrix.ruby}}
+        bundler-cache: true # 'bundle install' and cache
+    - name: Compile C-extension
+      run: EXTRALITE_BUNDLE=1 bundle exec rake compile
+    - name: Run tests
+      run:  bundle exec rake test

data/.github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: Tests
+name: Tests (extralite)
 
 on: [push, pull_request]
 
@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        ruby: ['2.7', '3.0', '3.1', '3.2', '3.3']
+        ruby: ['3.0', '3.1', '3.2', '3.3']
 
     name: >-
       ${{matrix.os}}, ${{matrix.ruby}}
@@ -28,13 +28,3 @@ jobs:
       run: bundle exec rake compile
     - name: Run tests
       run:  bundle exec rake test
-    - name: Build bundled gem
-      run:  bundle exec rake build_bundled
-    - name: Run tests with bundled gem
-      run: |
-        echo Installing extralite-bundle...
-        bundle config --local frozen false
-        mv Gemfile-bundle Gemfile
-        bundle install
-        echo Running tests...
-        bundle exec rake test

data/CHANGELOG.md

@@ -1,14 +1,42 @@
+# 2.5 2024-01-16
+
+- Update bundled sqlite to version 3.45.0
+- Implement `Database#batch_query` and related methods
+  [53](https://github.com/digital-fabric/extralite/issues/53)
+- Accept more options in `Database#initialize`
+  [48](https://github.com/digital-fabric/extralite/issues/48)
+- Fix `Database#pragma` to return single value when reading pragma value
+- Accept database name in `Database#tables` method
+- Improve `Database#batch_execute` - now accepts Enumerable and Callable
+  parameters [52](https://github.com/digital-fabric/extralite/issues/52)
+- Rename `Database#execute_multi` to `Database#batch_execute`
+- Implement Query#clone
+  [51](https://github.com/digital-fabric/extralite/issues/51)
+- Add support for GC compaction
+- Remove support for Ruby 2.7
+- Implement Query#<< [49](https://github.com/digital-fabric/extralite/issues/49)
+- Allow passing parameters in array
+- Add support for ractors
+  [#50](https://github.com/digital-fabric/extralite/issues/50)
+
 # 2.4 2023-12-24
 
-- Implement GVL release threshold. [#46](https://github.com/digital-fabric/extralite/pull/46)
+- Implement GVL release threshold.
+  [#46](https://github.com/digital-fabric/extralite/pull/46)
 - Implement write barriers for better GC performance.
-- Add support for binding large numbers and symbols. [#43](https://github.com/digital-fabric/extralite/pull/43)
-- Implement Database#transaction. [#42](https://github.com/digital-fabric/extralite/pull/42)
-- Add support for binding BLOBs. [#40](https://github.com/digital-fabric/extralite/pull/40)
+- Add support for binding large numbers and symbols.
+  [#43](https://github.com/digital-fabric/extralite/pull/43)
+- Implement Database#transaction.
+  [#42](https://github.com/digital-fabric/extralite/pull/42)
+- Add support for binding BLOBs.
+  [#40](https://github.com/digital-fabric/extralite/pull/40)
 - Minor fixes and cleanup of C-code.
-- Fix `Database#inspect` for a closed database instance [#37](https://github.com/digital-fabric/extralite/issues/37)
-- Add support for binding named parameters from Struct and Data classes [#30](https://github.com/digital-fabric/extralite/pull/30)
-- Update bundled SQLite code to version 3.44.2 [#32](https://github.com/digital-fabric/extralite/pull/32)
+- Fix `Database#inspect` for a closed database instance
+  [#37](https://github.com/digital-fabric/extralite/issues/37)
+- Add support for binding named parameters from Struct and Data classes
+  [#30](https://github.com/digital-fabric/extralite/pull/30)
+- Update bundled SQLite code to version 3.44.2
+  [#32](https://github.com/digital-fabric/extralite/pull/32)
 
 # 2.3 2023-11-12
 
@@ -28,7 +56,8 @@
 
 - Fix Sequel migrations (#8)
 - Redesign prepared statement functionality (#24)
-  - Rewrite `Extralite::PreparedStatement` into `Extralite::Query` with breaking API changes
+  - Rewrite `Extralite::PreparedStatement` into `Extralite::Query` with breaking
+    API changes
   - Add `Extralite::Iterator` class for external iteration
   - Add `Query#each_xxx`, `Query#to_a_xxx` method
   - Add `Query#eof?` method
@@ -77,7 +106,8 @@
 # 1.20 2023-01-21
 
 - Fix compilation error (#15 @sitano)
-- Add status methods `Extralite.runtime_status`, `Database#status`, `PreparedStatement#status` (#14 @sitano)
+- Add status methods `Extralite.runtime_status`, `Database#status`,
+  `PreparedStatement#status` (#14 @sitano)
 - Add `Database#interrupt` (#13 @sitano)
 - Add `Database#backup` (#11 @sitano)
 - Derive `Extralite::Error` from `StandardError` (#10 @sitano)
@@ -199,4 +229,3 @@
 ## 0.1 2021-05-21
 
 - First release
-

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    extralite (2.4)
+    extralite (2.5)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -49,6 +49,8 @@ gem 'extralite'
 
 You can also run `gem install extralite` if you just want to check it out.
 
+__Note__: Extralite supports Ruby 3.0 and higher.
+
 ### Installing the Extralite-SQLite3 Bundle
 
 If you don't have sqlite3 installed on your system, do not want to use the
@@ -102,9 +104,12 @@ db.query_single_value("select 'foo'") #=> "foo"
 
 # parameter binding (works for all query_xxx methods)
 db.query_hash('select ? as foo, ? as bar', 1, 2) #=> [{ :foo => 1, :bar => 2 }]
-db.query_hash('select ?2 as foo, ?1 as bar, ?1 * ?2 as baz', 6, 7) #=> [{ :foo => 7, :bar => 6, :baz => 42 }]
+db.query_hash('select ? as foo, ? as bar', [1, 2]) #=> [{ :foo => 1, :bar => 2 }]
+
+# explicit parameter indexes
+db.query_hash('select ?2 as foo, ?1 as bar, ?1 * ?2 as baz', 6, 7) #=> [{ :foo => 7, :bar => 6, :baz => 42 }
 
-# parameter binding of named parameters
+# named parameters
 db.query('select * from foo where bar = :bar', bar: 42)
 db.query('select * from foo where bar = :bar', 'bar' => 42)
 db.query('select * from foo where bar = :bar', ':bar' => 42)
@@ -121,8 +126,15 @@ db.execute('insert into foo values (?)', Extralite::Blob.new('Hello, 世界!'))
 db.execute('insert into foo values (?)', 'Hello, 世界!'.force_encoding(Encoding::ASCII_8BIT))
 
 # insert multiple rows
-db.execute_multi('insert into foo values (?)', ['bar', 'baz'])
-db.execute_multi('insert into foo values (?, ?)', [[1, 2], [3, 4]])
+db.batch_execute('insert into foo values (?)', ['bar', 'baz'])
+db.batch_execute('insert into foo values (?, ?)', [[1, 2], [3, 4]])
+
+# batch execute from enumerable
+db.batch_execute('insert into foo values (?)', 1..10)
+
+# batch execute from block
+source = [[1, 2], [2, 3], [3, 4]]
+db.batch_execute('insert into foo values (?, ?)') { source.shift }
 
 # prepared queries
 query = db.prepare('select ? as foo, ? as bar') #=> Extralite::Query
@@ -332,11 +344,18 @@ p articles.to_a
 
 ## Concurrency
 
-Extralite releases the GVL while making calls to the sqlite3 library that might
-block, such as when backing up a database, or when preparing a query. Extralite
-also releases the GVL periodically when iterating over records. By default, the
-GVL is released every 1000 records iterated. The GVL release threshold can be
-set separately for each database:
+### The Ruby GVL
+
+Extralite releases the [Ruby
+GVL](https://www.speedshop.co/2020/05/11/the-ruby-gvl-and-scaling.html) while
+making calls to the sqlite3 library that might block, such as when backing up a
+database, or when preparing a query. This allows other threads to run while the
+underlying sqlite3 library is busy preparing queries, fetching records and
+backing up databases.
+
+Extralite also releases the GVL periodically when iterating over records. By
+default, the GVL is released every 1000 records iterated. The GVL release
+threshold can be set separately for each database:
 
 ```ruby
 db.gvl_release_threshold = 10 # release GVL every 10 records
@@ -345,8 +364,8 @@ db.gvl_release_threshold = nil # use default value (currently 1000)
 ```
 
 For most applications, there's no need to tune the GVL threshold value, as it
-provides [excellent](#performance) performance characteristics for both single-threaded and
-multi-threaded applications.
+provides [excellent](#performance) performance characteristics for both
+single-threaded and multi-threaded applications.
 
 In a heavily multi-threaded application, releasing the GVL more often (lower
 threshold value) will lead to less latency (for threads not running a query),
@@ -360,6 +379,21 @@ latency and throughput:
 less latency & throughput <<< GVL release threshold >>> more latency & throughput
 ```
 
+### Thread Safety
+
+A single database instance can be safely used in multiple threads simultaneously
+as long as the following conditions are met:
+
+- No explicit transactions are used.
+- Each thread issues queries by calling `Database#query_xxx`, or uses a separate
+  `Query` instance.
+- The GVL release threshold is not `0` (i.e. the GVL is released periodically
+  while running queries.)
+
+### Use with Ractors
+
+Extralite databases can be used inside ractors 
+
 ## Performance
 
 A benchmark script is included, creating a table of various row counts, then

data/TODO.md

@@ -1,7 +1,5 @@
-- Improve tracing
 - Transactions and savepoints:
 
-  - `DB#transaction {}` - does a `BEGIN..COMMIT` - non-reentrant!
   - `DB#savepoint(name)` - creates a savepoint
   - `DB#release(name)` - releases a savepoint
   - `DB#rollback` - raises `Extralite::Rollback`, which is rescued by `DB#transaction`
@@ -10,7 +8,6 @@
 - More database methods:
 
   - `Database#quote`
-  - `Database#busy_timeout=` https://sqlite.org/c3ref/busy_timeout.html
   - `Database#cache_flush` https://sqlite.org/c3ref/db_cacheflush.html
   - `Database#release_memory` https://sqlite.org/c3ref/db_release_memory.html
 

data/ext/extralite/common.c

@@ -31,7 +31,7 @@ static inline VALUE get_column_value(sqlite3_stmt *stmt, int col, int type) {
   return Qnil;
 }
 
-void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value);
+int bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value);
 
 static inline void bind_key_value(sqlite3_stmt *stmt, VALUE k, VALUE v) {
   switch (TYPE(k)) {
@@ -72,24 +72,24 @@ void bind_struct_parameter_values(sqlite3_stmt *stmt, VALUE struct_obj) {
   RB_GC_GUARD(members);
 }
 
-inline void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value) {
+inline int bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value) {
   switch (TYPE(value)) {
     case T_NIL:
       sqlite3_bind_null(stmt, pos);
-      return;
+      return 1;
     case T_FIXNUM:
     case T_BIGNUM:
       sqlite3_bind_int64(stmt, pos, NUM2LL(value));
-      return;
+      return 1;
     case T_FLOAT:
       sqlite3_bind_double(stmt, pos, NUM2DBL(value));
-      return;
+      return 1;
     case T_TRUE:
       sqlite3_bind_int(stmt, pos, 1);
-      return;
+      return 1;
     case T_FALSE:
       sqlite3_bind_int(stmt, pos, 0);
-      return;
+      return 1;
     case T_SYMBOL:
       value = rb_sym2str(value);
     case T_STRING:
@@ -97,13 +97,20 @@ inline void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value) {
         sqlite3_bind_blob(stmt, pos, RSTRING_PTR(value), RSTRING_LEN(value), SQLITE_TRANSIENT);
       else
         sqlite3_bind_text(stmt, pos, RSTRING_PTR(value), RSTRING_LEN(value), SQLITE_TRANSIENT);
-      return;
+      return 1;
+    case T_ARRAY:
+      {
+        int count = RARRAY_LEN(value);
+        for (int i = 0; i < count; i++)
+          bind_parameter_value(stmt, pos + i, RARRAY_AREF(value, i));
+        return count;
+      }
     case T_HASH:
       bind_hash_parameter_values(stmt, value);
-      return;
+      return 0;
     case T_STRUCT:
       bind_struct_parameter_values(stmt, value);
-      return;
+      return 0;
     default:
       rb_raise(cParameterError, "Cannot bind parameter at position %d of type %"PRIsVALUE"",
         pos, rb_class_name(rb_obj_class(value)));
@@ -111,14 +118,18 @@ inline void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value) {
 }
 
 inline void bind_all_parameters(sqlite3_stmt *stmt, int argc, VALUE *argv) {
-  for (int i = 0; i < argc; i++) bind_parameter_value(stmt, i + 1, argv[i]);
+  int pos = 1;
+  for (int i = 0; i < argc; i++) {
+    pos += bind_parameter_value(stmt, pos, argv[i]);
+  }
 }
 
 inline void bind_all_parameters_from_object(sqlite3_stmt *stmt, VALUE obj) {
   if (TYPE(obj) == T_ARRAY) {
+    int pos = 1;
     int count = RARRAY_LEN(obj);
     for (int i = 0; i < count; i++)
-      bind_parameter_value(stmt, i + 1, RARRAY_AREF(obj, i));
+      pos += bind_parameter_value(stmt, pos, RARRAY_AREF(obj, i));
   }
   else
     bind_parameter_value(stmt, 1, obj);
@@ -423,8 +434,80 @@ VALUE safe_query_single_value(query_ctx *ctx) {
   return value;
 }
 
-VALUE safe_execute_multi(query_ctx *ctx) {
+enum batch_mode {
+  BATCH_EXECUTE,
+  BATCH_QUERY_ARY,
+  BATCH_QUERY_HASH,
+  BATCH_QUERY_SINGLE_COLUMN
+};
+
+static inline VALUE batch_iterate_hash(query_ctx *ctx) {
+  VALUE rows = rb_ary_new();
+  VALUE row = Qnil;
+  int column_count = sqlite3_column_count(ctx->stmt);
+  VALUE column_names = get_column_names(ctx->stmt, column_count);
+
+  while (stmt_iterate(ctx)) {
+    row = row_to_hash(ctx->stmt, column_count, column_names);
+    rb_ary_push(rows, row);
+  }
+
+  RB_GC_GUARD(column_names);
+  RB_GC_GUARD(rows);
+  return rows;
+}
+
+static inline VALUE batch_iterate_ary(query_ctx *ctx) {
+  VALUE rows = rb_ary_new();
+  VALUE row = Qnil;
+  int column_count = sqlite3_column_count(ctx->stmt);
+
+  while (stmt_iterate(ctx)) {
+    row = row_to_ary(ctx->stmt, column_count);
+    rb_ary_push(rows, row);
+  }
+
+  RB_GC_GUARD(rows);
+  return rows;
+}
+
+static inline VALUE batch_iterate_single_column(query_ctx *ctx) {
+  VALUE rows = rb_ary_new();
+  VALUE value = Qnil;
+  int column_count = sqlite3_column_count(ctx->stmt);
+  if (column_count != 1) rb_raise(cError, "Expected query result to have 1 column");
+
+  while (stmt_iterate(ctx)) {
+    value = get_column_value(ctx->stmt, 0, sqlite3_column_type(ctx->stmt, 0));
+    rb_ary_push(rows, value);
+  }
+
+  RB_GC_GUARD(rows);
+  return rows;
+}
+
+static inline void batch_iterate(query_ctx *ctx, enum batch_mode mode, VALUE *rows) {
+  switch (mode) {
+    case BATCH_EXECUTE:
+      while (stmt_iterate(ctx));
+      break;
+    case BATCH_QUERY_ARY:
+      *rows = batch_iterate_ary(ctx);
+      break;
+    case BATCH_QUERY_HASH:
+      *rows = batch_iterate_hash(ctx);
+      break;
+    case BATCH_QUERY_SINGLE_COLUMN:
+      *rows = batch_iterate_single_column(ctx);
+      break;
+  }
+}
+
+static inline VALUE batch_run_array(query_ctx *ctx, enum batch_mode mode) {
   int count = RARRAY_LEN(ctx->params);
+  int block_given = rb_block_given_p();
+  VALUE results = (mode != BATCH_EXECUTE) && !block_given ? rb_ary_new() : Qnil;
+  VALUE rows = Qnil;
   int changes = 0;
 
   for (int i = 0; i < count; i++) {
@@ -432,11 +515,135 @@ VALUE safe_execute_multi(query_ctx *ctx) {
     sqlite3_clear_bindings(ctx->stmt);
     bind_all_parameters_from_object(ctx->stmt, RARRAY_AREF(ctx->params, i));
 
-    while (stmt_iterate(ctx));
+    batch_iterate(ctx, mode, &rows);
     changes += sqlite3_changes(ctx->sqlite3_db);
+
+    if (mode != BATCH_EXECUTE) {
+      if (block_given)
+        rb_yield(rows);
+      else
+        rb_ary_push(results, rows);
+    }
+  }
+
+  RB_GC_GUARD(rows);
+  RB_GC_GUARD(results);
+
+  if (mode == BATCH_EXECUTE || block_given)
+    return INT2FIX(changes);
+  else
+    return results;
+}
+
+struct batch_execute_each_ctx {
+  query_ctx *ctx;
+  enum batch_mode mode;
+  int block_given;
+  VALUE results;
+  int changes;
+};
+
+static VALUE batch_run_each_iter(RB_BLOCK_CALL_FUNC_ARGLIST(yield_value, vctx)) {
+  struct batch_execute_each_ctx *each_ctx = (struct batch_execute_each_ctx*)vctx;
+  VALUE rows = Qnil;
+
+  sqlite3_reset(each_ctx->ctx->stmt);
+  sqlite3_clear_bindings(each_ctx->ctx->stmt);
+  bind_all_parameters_from_object(each_ctx->ctx->stmt, yield_value);
+
+  batch_iterate(each_ctx->ctx, each_ctx->mode, &rows);
+  each_ctx->changes += sqlite3_changes(each_ctx->ctx->sqlite3_db);
+
+  if (each_ctx->mode != BATCH_EXECUTE) {
+    if (each_ctx->block_given)
+      rb_yield(rows);
+    else
+      rb_ary_push(each_ctx->results, rows);
+  }
+  RB_GC_GUARD(rows);
+
+  return Qnil;
+}
+
+static inline VALUE batch_run_each(query_ctx *ctx, enum batch_mode mode) {
+  struct batch_execute_each_ctx each_ctx = {
+    .ctx          = ctx,
+    .mode         = mode,
+    .block_given  = rb_block_given_p(),
+    .results      = ((mode != BATCH_EXECUTE) && !rb_block_given_p() ? rb_ary_new() : Qnil),
+    .changes      = 0
+  };
+  rb_block_call(ctx->params, ID_each, 0, 0, batch_run_each_iter, (VALUE)&each_ctx);
+
+  if (mode == BATCH_EXECUTE || each_ctx.block_given)
+    return INT2FIX(each_ctx.changes);
+  else
+    return each_ctx.results;
+}
+
+static inline VALUE batch_run_proc(query_ctx *ctx, enum batch_mode mode) {
+  VALUE params = Qnil;
+  int block_given = rb_block_given_p();
+  VALUE results = (mode != BATCH_EXECUTE) && !block_given ? rb_ary_new() : Qnil;
+  VALUE rows = Qnil;
+  int changes = 0;
+
+  while (1) {
+    params = rb_funcall(ctx->params, ID_call, 0);
+    if (NIL_P(params)) break;
+
+    sqlite3_reset(ctx->stmt);
+    sqlite3_clear_bindings(ctx->stmt);
+    bind_all_parameters_from_object(ctx->stmt, params);
+
+    batch_iterate(ctx, mode, &rows);
+    changes += sqlite3_changes(ctx->sqlite3_db);
+
+    if (mode != BATCH_EXECUTE) {
+      if (block_given)
+        rb_yield(rows);
+      else
+        rb_ary_push(results, rows);
+    }
   }
 
-  return INT2FIX(changes);
+  RB_GC_GUARD(rows);
+  RB_GC_GUARD(results);
+  RB_GC_GUARD(params);
+
+  if (mode == BATCH_EXECUTE || block_given)
+    return INT2FIX(changes);
+  else
+    return results;
+}
+
+static inline VALUE batch_run(query_ctx *ctx, enum batch_mode mode) {
+  if (TYPE(ctx->params) == T_ARRAY)
+    return batch_run_array(ctx, mode);
+  
+  if (rb_respond_to(ctx->params, ID_each))
+    return batch_run_each(ctx, mode);
+  
+  if (rb_respond_to(ctx->params, ID_call))
+    return batch_run_proc(ctx, mode);
+  
+  rb_raise(cParameterError, "Invalid parameter source supplied to #batch_execute");
+}
+
+VALUE safe_batch_execute(query_ctx *ctx) {
+  return batch_run(ctx, BATCH_EXECUTE);
+}
+
+VALUE safe_batch_query(query_ctx *ctx) {
+  return batch_run(ctx, BATCH_QUERY_HASH);
+}
+
+VALUE safe_batch_query_ary(query_ctx *ctx) {
+  return batch_run(ctx, BATCH_QUERY_ARY);
+}
+
+VALUE safe_batch_query_single_column(query_ctx *ctx) {
+  return batch_run(ctx, BATCH_QUERY_SINGLE_COLUMN);
 }
 
 VALUE safe_query_columns(query_ctx *ctx) {