extralite 1.4 → 1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b112a3c34429f8b44c4b4a922c4e038387f6c1db54e6f01377200d7a60324dd4
-  data.tar.gz: a664bc81719347079fdfbbffdf2e665bd2772fe8dcf9c30b3ec18c6238a8888a
+  metadata.gz: ad8337aebac5ef75132340d8336389fe0c927ed454b5792d57f80efa4f362664
+  data.tar.gz: 602ad3648312e810398fac215353b0fac5aa916ff7a3a7901ba1990c9e3fb145
 SHA512:
-  metadata.gz: '053783bb3f186130fc92b0a57403f0f9472a98e6ce45b1fbb128c5e43914b2a6ec56d39a5e6b10e16f5e6b46f8e9d586837860f63e7538ad238fc0a7aa0995c3'
-  data.tar.gz: '058538866f9b2c6a5a0116c8ed3a801aa7bbb8ef8f911eff6d94d7a9cfa98861ad98f4d27e06a736b18c4b62e3c2369a4a306492cc562693da0282363e00f208'
+  metadata.gz: a5831721c27d2eb27c8433bcd286ba826861946b7894eb2c1564f449d321d2eaeec7cce13a4166b10db799f9e9e391bc2ba65961edc7b144592d1994767e69b0
+  data.tar.gz: ce77ee584ec89bae2819a2c0384ca7cc08daf00f405d0bf379f322d6182b3bf55bceb0ac0136d52fddd3fe5a43e4e48eb23cbfc00eef74a0439ce25eeeb529f7

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 1.5 2021-12-13
+
+- Release GVL while preparing statements
+- Use `sqlite3_prepare_v2` instead of deprecated `sqlite_prepare`
+
 ## 1.4 2021-08-25
 
 - Fix possible segfault in cleanup_stmt

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    extralite (1.4)
+    extralite (1.5)
 
 GEM
   remote: https://rubygems.org/

data/ext/extralite/extralite.c

@@ -1,5 +1,6 @@
 #include <stdio.h>
 #include "ruby.h"
+#include "ruby/thread.h"
 #include <sqlite3.h>
 
 VALUE cError;
@@ -109,7 +110,7 @@ inline VALUE get_column_value(sqlite3_stmt *stmt, int col, int type) {
 }
 
 static inline void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value) {
-  switch (TYPE(value)) { 
+  switch (TYPE(value)) {
     case T_NIL:
       sqlite3_bind_null(stmt, pos);
       return;
@@ -143,7 +144,7 @@ static inline void bind_all_parameters(sqlite3_stmt *stmt, int argc, VALUE *argv
 
 static inline VALUE get_column_names(sqlite3_stmt *stmt, int column_count) {
   VALUE arr = rb_ary_new2(column_count);
-  for (int i = 0; i < column_count; i++) { 
+  for (int i = 0; i < column_count; i++) {
     VALUE name = ID2SYM(rb_intern(sqlite3_column_name(stmt, i)));
     rb_ary_push(arr, name);
   }
@@ -168,29 +169,62 @@ static inline VALUE row_to_ary(sqlite3_stmt *stmt, int column_count) {
   return row;
 }
 
-inline void prepare_multi_stmt(sqlite3 *db, sqlite3_stmt **stmt, VALUE sql) {
-  const char *rest = 0;
-  const char *ptr = RSTRING_PTR(sql);
-  const char *end = ptr + RSTRING_LEN(sql);
+struct multi_stmt_ctx {
+  sqlite3 *db;
+  sqlite3_stmt **stmt;
+  const char *str;
+  int len;
+  int rc;
+};
+
+void *prepare_multi_stmt_without_gvl(void *ptr) {
+  struct multi_stmt_ctx *ctx = (struct multi_stmt_ctx *)ptr;
+  const char *rest = NULL;
+  const char *str = ctx->str;
+  const char *end = ctx->str + ctx->len;
   while (1) {
-    int rc = sqlite3_prepare(db, ptr, end - ptr, stmt, &rest);
-    if (rc) {
-      sqlite3_finalize(*stmt);
-      rb_raise(cSQLError, "%s", sqlite3_errmsg(db));
+    ctx->rc = sqlite3_prepare_v2(ctx->db, str, end - str, ctx->stmt, &rest);
+    if (ctx->rc) {
+      sqlite3_finalize(*ctx->stmt);
+      return NULL;
     }
 
-    if (rest == end) return;
-    
+    if (rest == end) return NULL;
+
     // perform current query, but discard its results
-    rc = sqlite3_step(*stmt);
-    sqlite3_finalize(*stmt);
-    switch (rc) {
+    ctx->rc = sqlite3_step(*ctx->stmt);
+    sqlite3_finalize(*ctx->stmt);
+    switch (ctx->rc) {
     case SQLITE_BUSY:
-      rb_raise(cBusyError, "Database is busy");
     case SQLITE_ERROR:
-      rb_raise(cSQLError, "%s", sqlite3_errmsg(db));
+    case SQLITE_MISUSE:
+      return NULL;
     }
-    ptr = rest;
+    str = rest;
+  }
+  return NULL;
+}
+
+/*
+This function prepares a statement from an SQL string containing one or more SQL
+statements. It will release the GVL while the statements are being prepared and
+executed. All statements excluding the last one are executed. The last statement
+is not executed, but instead handed back to the caller for looping over results.
+*/
+inline void prepare_multi_stmt(sqlite3 *db, sqlite3_stmt **stmt, VALUE sql) {
+  struct multi_stmt_ctx ctx = {db, stmt, RSTRING_PTR(sql), RSTRING_LEN(sql), 0};
+  rb_thread_call_without_gvl(prepare_multi_stmt_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
+  RB_GC_GUARD(sql);
+
+  switch (ctx.rc) {
+  case 0:
+    return;
+  case SQLITE_BUSY:
+    rb_raise(cBusyError, "Database is busy");
+  case SQLITE_ERROR:
+    rb_raise(cSQLError, "%s", sqlite3_errmsg(db));
+  default:
+    rb_raise(cError, "Invalid return code for prepare_multi_stmt_without_gvl: %d (please open an issue on https://github.com/digital-fabric/extralite)", ctx.rc);
   }
 }
 
@@ -207,7 +241,7 @@ inline int stmt_iterate(sqlite3_stmt *stmt, sqlite3 *db) {
     case SQLITE_ERROR:
       rb_raise(cSQLError, "%s", sqlite3_errmsg(db));
     default:
-      rb_raise(cError, "Invalid return code for sqlite3_step: %d", rc);
+      rb_raise(cError, "Invalid return code for sqlite3_step: %d (please open an issue on https://github.com/digital-fabric/extralite)", rc);
   }
 
   return 0;
@@ -292,7 +326,7 @@ VALUE safe_query_ary(VALUE arg) {
     row = row_to_ary(ctx->stmt, column_count);
     if (yield_to_block) rb_yield(row); else rb_ary_push(result, row);
   }
-  
+
   RB_GC_GUARD(row);
   RB_GC_GUARD(result);
   return result;
@@ -453,14 +487,14 @@ void Init_Extralite() {
   rb_define_method(cDatabase, "initialize", Database_initialize, 1);
   rb_define_method(cDatabase, "close", Database_close, 0);
   rb_define_method(cDatabase, "closed?", Database_closed_p, 0);
-  
+
   rb_define_method(cDatabase, "query", Database_query_hash, -1);
   rb_define_method(cDatabase, "query_hash", Database_query_hash, -1);
   rb_define_method(cDatabase, "query_ary", Database_query_ary, -1);
   rb_define_method(cDatabase, "query_single_row", Database_query_single_row, -1);
   rb_define_method(cDatabase, "query_single_column", Database_query_single_column, -1);
   rb_define_method(cDatabase, "query_single_value", Database_query_single_value, -1);
-  
+
   rb_define_method(cDatabase, "last_insert_rowid", Database_last_insert_rowid, 0);
   rb_define_method(cDatabase, "changes", Database_changes, 0);
   rb_define_method(cDatabase, "filename", Database_filename, -1);

data/lib/extralite/version.rb

@@ -1,3 +1,3 @@
 module Extralite
-  VERSION = '1.4'
+  VERSION = '1.5'
 end

data/test/perf.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'sqlite3'
+  gem 'extralite', path: '..'
+  gem 'benchmark-ips'
+end
+
+require 'benchmark/ips'
+require 'fileutils'
+
+DB_PATH = '/tmp/extralite_sqlite3_perf.db'
+COUNT = 10000
+
+def prepare_database
+  FileUtils.rm(DB_PATH) rescue nil
+  db = Extralite::Database.new(DB_PATH)
+  db.query('create table foo ( a integer primary key, b text )')
+  db.query('begin')
+  COUNT.times { db.query('insert into foo (b) values (?)', "hello#{rand(1000)}" )}
+  db.query('commit')
+end
+
+def sqlite3_run
+  db = SQLite3::Database.new(DB_PATH, :results_as_hash => true)
+  results = db.execute('select * from foo')
+  raise unless results.size == COUNT
+end
+
+def extralite_run
+  db = Extralite::Database.new(DB_PATH)
+  results = db.query('select * from foo')
+  raise unless results.size == COUNT
+end
+
+prepare_database
+
+Benchmark.ips do |x|
+  x.config(:time => 3, :warmup => 1)
+
+  x.report("sqlite3") { sqlite3_run }
+  x.report("extralite") { extralite_run }
+
+  x.compare!
+end

data/test/test_database.rb

@@ -4,7 +4,7 @@ require_relative 'helper'
 
 class DatabaseTest < MiniTest::Test
   def setup
-    @db = Extralite::Database.new('/tmp/extralite.db')
+    @db = Extralite::Database.new(':memory:')
     @db.query('create table if not exists t (x,y,z)')
     @db.query('delete from t')
     @db.query('insert into t values (1, 2, 3)')
@@ -55,7 +55,7 @@ class DatabaseTest < MiniTest::Test
   def test_query_single_column
     r = @db.query_single_column('select y from t')
     assert_equal [2, 5], r
-  
+
     r = @db.query_single_column('select y from t where x = 2')
     assert_equal [], r
 end
@@ -82,6 +82,17 @@ end
     assert_equal [1, 4, 'a', 'd'], @db.query_single_column('select x from t order by x')
   end
 
+  def test_multiple_statements_with_error
+    error = nil
+    begin
+      @db.query("insert into t values foo; insert into t values ('d', 'e', 'f');")
+    rescue => error
+    end
+
+    assert_kind_of Extralite::SQLError, error
+    assert_equal 'near "foo": syntax error', error.message
+  end
+
   def test_empty_sql
     r = @db.query(' ')
     assert_nil r
@@ -97,7 +108,75 @@ end
 
     assert_equal @db, @db.close
     assert_equal true, @db.closed?
-    
+
     assert_raises(Extralite::Error) { @db.query_single_value('select 42') }
   end
 end
+
+class ScenarioTest < MiniTest::Test
+  def setup
+    @db = Extralite::Database.new('/tmp/extralite.db')
+    @db.query('create table if not exists t (x,y,z)')
+    @db.query('delete from t')
+    @db.query('insert into t values (1, 2, 3)')
+    @db.query('insert into t values (4, 5, 6)')
+  end
+
+  def test_concurrent_transactions
+    done = false
+    t = Thread.new do
+      db = Extralite::Database.new('/tmp/extralite.db')
+      db.query 'begin immediate'
+      sleep 0.01 until done
+
+      while true
+        begin
+          db.query 'commit'
+          break
+        rescue Extralite::BusyError
+          sleep 0.01
+        end
+      end
+    end
+
+    sleep 0.1
+    @db.query 'begin deferred'
+    result = @db.query_single_column('select x from t')
+    assert_equal [1, 4], result
+
+    assert_raises(Extralite::BusyError) do
+      @db.query('insert into t values (7, 8, 9)')
+    end
+
+    done = true
+    sleep 0.1
+
+    assert_raises(Extralite::BusyError) do
+      @db.query('insert into t values (7, 8, 9)')
+    end
+
+    assert_equal true, @db.transaction_active?
+
+    # the thing to do in this case is to commit the read transaction, allowing
+    # the other thread to commit its write transaction, and then we can
+    # "upgrade" to a write transaction
+
+    @db.query('commit')
+
+    while true
+      begin
+        @db.query('begin immediate')
+        break
+      rescue Extralite::BusyError
+        sleep 0.1
+      end
+    end
+
+    @db.query('insert into t values (7, 8, 9)')
+    @db.query('commit')
+
+    result = @db.query_single_column('select x from t')
+    assert_equal [1, 4, 7], result
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: extralite
 version: !ruby/object:Gem::Version
-  version: '1.4'
+  version: '1.5'
 platform: ruby
 authors:
 - Sharon Rosner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-25 00:00:00.000000000 Z
+date: 2021-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -104,6 +104,7 @@ files:
 - lib/extralite.rb
 - lib/extralite/version.rb
 - test/helper.rb
+- test/perf.rb
 - test/test_database.rb
 homepage: https://github.com/digital-fabric/extralite
 licenses: