extralite 2.3 → 2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 130809d0ec92b7ae91e31f05e24c072500ef3096bda542a293469f1e206ff8eb
-  data.tar.gz: b7236d13577003ed2ccd806eb31b3a6703d3126196147f78220504b7fd0264b8
+  metadata.gz: d8ccbfbf37744a5b90d30038c3ba4e73f7d2a094a0f18f21cd8b58d66ec32b42
+  data.tar.gz: 5c6d3f9c967f7056f8eeb7ba471e2db50a26be334bdeec7ddf80dbd521efff77
 SHA512:
-  metadata.gz: 44b5ab1374077c97418b14581947aaa2dd20010590e21beaa755be9aaa6228b1408e5593c69f9f901f7935b0ba58494772cd9c0062b7632dc8c48d4ea54845bd
-  data.tar.gz: 218b35977b8ce307b9e738dfb280bfcfa4e67910542d03cf11aaec36acbaf61ba74db5c10f5b2d21364384c9cfbc830f91f945d5417dd00b6b090895dadfe276
+  metadata.gz: 19f10ce33c25e3b0837ec452b8eff0c4c433a729c79162d8b73e65411142f6df61703e0d7c254ac269fbf02ae39aa1c0cde5cb8f9fe3f262ab9ad0af503ceb99
+  data.tar.gz: 921aeeb63e23bfc145af2c978fb3ce5a47b643cf23e265aaf581eb067e8a76ef5c63effde30c567d7565a19fa8f844868456f996d5413a70ef2ba774a927d638

data/.editorconfig

@@ -0,0 +1,6 @@
+[*]
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space
+indent_size = 2

data/.github/workflows/test.yml

@@ -2,6 +2,10 @@ name: Tests
 
 on: [push, pull_request]
 
+concurrency:
+  group: tests-${{ format('{0}-{1}', github.head_ref || github.run_number, github.job) }}
+  cancel-in-progress: true
+
 jobs:
   build:
     strategy:
@@ -15,7 +19,7 @@ jobs:
 
     runs-on: ${{matrix.os}}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{matrix.ruby}}
@@ -24,3 +28,13 @@ jobs:
       run: bundle exec rake compile
     - name: Run tests
       run:  bundle exec rake test
+    - name: Build bundled gem
+      run:  bundle exec rake build_bundled
+    - name: Run tests with bundled gem
+      run: |
+        echo Installing extralite-bundle...
+        bundle config --local frozen false
+        mv Gemfile-bundle Gemfile
+        bundle install
+        echo Running tests...
+        bundle exec rake test

data/.gitignore

@@ -10,6 +10,9 @@
 /test/version_tmp/
 /tmp/
 lib/extralite_ext.*
+Gemfile-bundle.lock
+/cmake-build-debug/
+CMakeLists.txt
 
 # Used by dotenv library to load environment variables.
 # .env

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+# 2.4 2023-12-24
+
+- Implement GVL release threshold. [#46](https://github.com/digital-fabric/extralite/pull/46)
+- Implement write barriers for better GC performance.
+- Add support for binding large numbers and symbols. [#43](https://github.com/digital-fabric/extralite/pull/43)
+- Implement Database#transaction. [#42](https://github.com/digital-fabric/extralite/pull/42)
+- Add support for binding BLOBs. [#40](https://github.com/digital-fabric/extralite/pull/40)
+- Minor fixes and cleanup of C-code.
+- Fix `Database#inspect` for a closed database instance [#37](https://github.com/digital-fabric/extralite/issues/37)
+- Add support for binding named parameters from Struct and Data classes [#30](https://github.com/digital-fabric/extralite/pull/30)
+- Update bundled SQLite code to version 3.44.2 [#32](https://github.com/digital-fabric/extralite/pull/32)
+
 # 2.3 2023-11-12
 
 - Update bundled SQLite to version 3.44.0 (#29)

data/Gemfile-bundle

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gemspec name: 'extralite'
+
+gem "extralite-bundle", git: "file://#{File.expand_path(__dir__)}", ref: `git rev-parse --abbrev-ref HEAD`.strip

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    extralite (2.3)
+    extralite (2.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
     docile (1.4.0)
-    json (2.6.3)
+    json (2.7.1)
     minitest (5.15.0)
     rake (13.1.0)
     rake-compiler (1.1.6)
@@ -34,4 +34,4 @@ DEPENDENCIES
   yard (= 0.9.27)
 
 BUNDLED WITH
-   2.3.3
+   2.4.22

data/README.md

@@ -14,14 +14,13 @@ with an SQLite3 database, as well as prepared queries (prepared statements).
 Extralite comes in two flavors: the `extralite` gem which uses the
 system-installed sqlite3 library, and the `extralite-bundle` gem which bundles
 the latest version of SQLite
-([3.44.0](https://sqlite.org/releaselog/3_44_0.html)), offering access to the
+([3.44.2](https://sqlite.org/releaselog/3_44_2.html)), offering access to the
 latest features and enhancements.
 
 ## Features
 
 - Super fast - [up to 11x faster](#performance) than the
-  [sqlite3](https://github.com/sparklemotion/sqlite3-ruby) gem (see also
-  [comparison](#why-not-just-use-the-sqlite3-gem).)
+  [sqlite3](https://github.com/sparklemotion/sqlite3-ruby) gem.
 - A variety of methods for different data access patterns: rows as hashes, rows
   as arrays, single row, single column, single value.
 - Prepared statements.
@@ -30,10 +29,12 @@ latest features and enhancements.
 - Use system-installed sqlite3, or the [bundled latest version of
   SQLite3](#installing-the-extralite-sqlite3-bundle).
 - Improved [concurrency](#concurrency) for multithreaded apps: the Ruby GVL is
-  released while preparing SQL statements and while iterating over results.
+  released peridically while preparing SQL statements and while iterating over
+  results.
 - Automatically execute SQL strings containing multiple semicolon-separated
   queries (handy for creating/modifying schemas).
-- Execute the same query with multiple parameter lists (useful for inserting records).
+- Execute the same query with multiple parameter lists (useful for inserting
+  records).
 - Load extensions (loading of extensions is autmatically enabled. You can find
   some useful extensions here: https://github.com/nalgeon/sqlean.)
 - Includes a [Sequel adapter](#usage-with-sequel).
@@ -101,12 +102,24 @@ db.query_single_value("select 'foo'") #=> "foo"
 
 # parameter binding (works for all query_xxx methods)
 db.query_hash('select ? as foo, ? as bar', 1, 2) #=> [{ :foo => 1, :bar => 2 }]
+db.query_hash('select ?2 as foo, ?1 as bar, ?1 * ?2 as baz', 6, 7) #=> [{ :foo => 7, :bar => 6, :baz => 42 }]
 
 # parameter binding of named parameters
 db.query('select * from foo where bar = :bar', bar: 42)
 db.query('select * from foo where bar = :bar', 'bar' => 42)
 db.query('select * from foo where bar = :bar', ':bar' => 42)
 
+# parameter binding of named parameters from Struct and Data
+SomeStruct = Struct.new(:foo, :bar)
+db.query_single_column('select :bar', SomeStruct.new(41, 42)) #=> [42]
+SomeData = Data.define(:foo, :bar)
+db.query_single_column('select :bar', SomeData.new(foo: 41, bar: 42)) #=> [42]
+
+# parameter binding for binary data (BLOBs)
+db.execute('insert into foo values (?)', File.binread('/path/to/file'))
+db.execute('insert into foo values (?)', Extralite::Blob.new('Hello, 世界!'))
+db.execute('insert into foo values (?)', 'Hello, 世界!'.force_encoding(Encoding::ASCII_8BIT))
+
 # insert multiple rows
 db.execute_multi('insert into foo values (?)', ['bar', 'baz'])
 db.execute_multi('insert into foo values (?, ?)', [[1, 2], [3, 4]])
@@ -166,6 +179,13 @@ db.pragma(:journal_mode) #=> 'wal'
 # load an extension
 db.load_extension('/path/to/extension.so')
 
+# run queries in a transaction
+db.transaction do
+  db.execute('insert into foo values (?)', 1)
+  db.execute('insert into foo values (?)', 2)
+  db.execute('insert into foo values (?)', 3)
+end
+
 # close database
 db.close
 db.closed? #=> true
@@ -194,6 +214,23 @@ ensure
 end
 ```
 
+### Running transactions
+
+In order to run multiple queries in a single transaction, use
+`Database#transaction`, passing a block that runs the queries. You can specify
+the [transaction
+mode](https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions).
+The default mode is `:immediate`:
+
+```ruby
+db.transaction { ... } # Run an immediate transaction
+db.transaction(:deferred) { ... } # Run a deferred transaction
+db.transaction(:exclusive) { ... } # Run an exclusive transaction
+```
+
+If an exception is raised in the given block, the transaction will be rolled
+back. Otherwise, it is committed.
+
 ### Creating Backups
 
 You can use `Database#backup` to create backup copies of a database. The
@@ -295,11 +332,33 @@ p articles.to_a
 
 ## Concurrency
 
-Extralite releases the GVL while making blocking calls to the sqlite3 library,
-that is while preparing SQL statements and fetching rows. Releasing the GVL
-allows other threads to run while the sqlite3 library is busy compiling SQL into
-bytecode, or fetching the next row. This *does not* hurt Extralite's
-performance, as you can see:
+Extralite releases the GVL while making calls to the sqlite3 library that might
+block, such as when backing up a database, or when preparing a query. Extralite
+also releases the GVL periodically when iterating over records. By default, the
+GVL is released every 1000 records iterated. The GVL release threshold can be
+set separately for each database:
+
+```ruby
+db.gvl_release_threshold = 10 # release GVL every 10 records
+
+db.gvl_release_threshold = nil # use default value (currently 1000)
+```
+
+For most applications, there's no need to tune the GVL threshold value, as it
+provides [excellent](#performance) performance characteristics for both single-threaded and
+multi-threaded applications.
+
+In a heavily multi-threaded application, releasing the GVL more often (lower
+threshold value) will lead to less latency (for threads not running a query),
+but will also hurt the throughput (for the thread running the query). Releasing
+the GVL less often (higher threshold value) will lead to better throughput for
+queries, while increasing latency for threads not running a query. The following
+diagram demonstrates the relationship between the GVL release threshold value,
+latency and throughput:
+
+```
+less latency & throughput <<< GVL release threshold >>> more latency & throughput
+```
 
 ## Performance
 

data/bin/update_sqlite_source

@@ -9,10 +9,10 @@ require 'date'
 
 FileUtils.cd '/tmp'
 
-version_id = version.gsub('.', '')
+version_id = version.split('.').each_with_index.map { |v, i| i == 0 ? v : v.rjust(2, '0') }.join
 version_id += '0' * (7 - version_id.length)
 url = "https://sqlite.org/#{Date.today.year}/sqlite-amalgamation-#{version_id}.zip"
-dest = File.expand_path('../ext/extralite', __dir__)
+dest = File.expand_path('../ext/sqlite3', __dir__)
 
 puts "Downloading from #{url}..."
 `curl #{url} > #{version_id}.zip`
@@ -23,4 +23,11 @@ puts "Unzipping zip file..."
 puts "Copying source files"
 `cp sqlite-amalgamation-#{version_id}/sqlite3.* #{dest}/`
 
-puts 'Done updating source files'
+puts "Updating README"
+readme_path = File.expand_path('../README.md', __dir__)
+readme = File.read(readme_path)
+readme.gsub!(/\[\d+\.\d+\.\d+\]/, "[#{version}]")
+readme.gsub!(/\d+_\d+_\d+\.html/, "#{version.gsub('.', '_')}.html")
+File.write(readme_path, readme)
+
+puts 'Done updating source files'

data/ext/extralite/common.c

@@ -3,6 +3,15 @@
 
 rb_encoding *UTF8_ENCODING;
 
+inline void *gvl_call(enum gvl_mode mode, void *(*fn)(void *), void *data) {
+  switch (mode) {
+    case GVL_RELEASE:
+      return rb_thread_call_without_gvl(fn, data, RUBY_UBF_IO, 0);
+    default:
+      return fn(data);
+  }
+}
+
 static inline VALUE get_column_value(sqlite3_stmt *stmt, int col, int type) {
   switch (type) {
     case SQLITE_NULL:
@@ -24,37 +33,52 @@ static inline VALUE get_column_value(sqlite3_stmt *stmt, int col, int type) {
 
 void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value);
 
+static inline void bind_key_value(sqlite3_stmt *stmt, VALUE k, VALUE v) {
+  switch (TYPE(k)) {
+    case T_FIXNUM:
+      bind_parameter_value(stmt, FIX2INT(k), v);
+      break;
+    case T_SYMBOL:
+      k = rb_sym2str(k);
+    case T_STRING:
+      if (RSTRING_PTR(k)[0] != ':') k = rb_str_plus(rb_str_new2(":"), k);
+      int pos = sqlite3_bind_parameter_index(stmt, StringValuePtr(k));
+      bind_parameter_value(stmt, pos, v);
+      break;
+    default:
+      rb_raise(cParameterError, "Cannot bind parameter with a key of type %"PRIsVALUE"",
+        rb_class_name(rb_obj_class(k)));
+  }
+}
+
 void bind_hash_parameter_values(sqlite3_stmt *stmt, VALUE hash) {
   VALUE keys = rb_funcall(hash, ID_keys, 0);
   long len = RARRAY_LEN(keys);
   for (long i = 0; i < len; i++) {
     VALUE k = RARRAY_AREF(keys, i);
     VALUE v = rb_hash_aref(hash, k);
-
-    switch (TYPE(k)) {
-      case T_FIXNUM:
-        bind_parameter_value(stmt, FIX2INT(k), v);
-        break;
-      case T_SYMBOL:
-        k = rb_funcall(k, ID_to_s, 0);
-      case T_STRING:
-        if(RSTRING_PTR(k)[0] != ':') k = rb_str_plus(rb_str_new2(":"), k);
-        int pos = sqlite3_bind_parameter_index(stmt, StringValuePtr(k));
-        bind_parameter_value(stmt, pos, v);
-        break;
-      default:
-        rb_raise(cError, "Cannot bind hash key value idx %ld", i);
-    }
+    bind_key_value(stmt, k, v);
   }
   RB_GC_GUARD(keys);
 }
 
+void bind_struct_parameter_values(sqlite3_stmt *stmt, VALUE struct_obj) {
+  VALUE members = rb_struct_members(struct_obj);
+  for (long i = 0; i < RSTRUCT_LEN(struct_obj); i++) {
+    VALUE k = rb_ary_entry(members, i);
+    VALUE v = RSTRUCT_GET(struct_obj, i);
+    bind_key_value(stmt, k, v);
+  }
+  RB_GC_GUARD(members);
+}
+
 inline void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value) {
   switch (TYPE(value)) {
     case T_NIL:
       sqlite3_bind_null(stmt, pos);
       return;
     case T_FIXNUM:
+    case T_BIGNUM:
       sqlite3_bind_int64(stmt, pos, NUM2LL(value));
       return;
     case T_FLOAT:
@@ -66,14 +90,23 @@ inline void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value) {
     case T_FALSE:
       sqlite3_bind_int(stmt, pos, 0);
       return;
+    case T_SYMBOL:
+      value = rb_sym2str(value);
     case T_STRING:
-      sqlite3_bind_text(stmt, pos, RSTRING_PTR(value), RSTRING_LEN(value), SQLITE_TRANSIENT);
+      if (rb_enc_get_index(value) == rb_ascii8bit_encindex() || CLASS_OF(value) == cBlob)
+        sqlite3_bind_blob(stmt, pos, RSTRING_PTR(value), RSTRING_LEN(value), SQLITE_TRANSIENT);
+      else
+        sqlite3_bind_text(stmt, pos, RSTRING_PTR(value), RSTRING_LEN(value), SQLITE_TRANSIENT);
       return;
     case T_HASH:
       bind_hash_parameter_values(stmt, value);
       return;
+    case T_STRUCT:
+      bind_struct_parameter_values(stmt, value);
+      return;
     default:
-      rb_raise(cError, "Cannot bind parameter at position %d", pos);
+      rb_raise(cParameterError, "Cannot bind parameter at position %d of type %"PRIsVALUE"",
+        pos, rb_class_name(rb_obj_class(value)));
   }
 }
 
@@ -126,7 +159,7 @@ typedef struct {
   int rc;
 } prepare_stmt_ctx;
 
-void *prepare_multi_stmt_without_gvl(void *ptr) {
+void *prepare_multi_stmt_impl(void *ptr) {
   prepare_stmt_ctx *ctx = (prepare_stmt_ctx *)ptr;
   const char *rest = NULL;
   const char *str = ctx->str;
@@ -163,7 +196,7 @@ is not executed, but instead handed back to the caller for looping over results.
 */
 void prepare_multi_stmt(sqlite3 *db, sqlite3_stmt **stmt, VALUE sql) {
   prepare_stmt_ctx ctx = {db, stmt, RSTRING_PTR(sql), RSTRING_LEN(sql), 0};
-  rb_thread_call_without_gvl(prepare_multi_stmt_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
+  gvl_call(GVL_RELEASE, prepare_multi_stmt_impl, (void *)&ctx);
   RB_GC_GUARD(sql);
 
   switch (ctx.rc) {
@@ -180,7 +213,7 @@ void prepare_multi_stmt(sqlite3 *db, sqlite3_stmt **stmt, VALUE sql) {
 
 #define SQLITE_MULTI_STMT -1
 
-void *prepare_single_stmt_without_gvl(void *ptr) {
+void *prepare_single_stmt_impl(void *ptr) {
   prepare_stmt_ctx *ctx = (prepare_stmt_ctx *)ptr;
   const char *rest = NULL;
   const char *str = ctx->str;
@@ -205,7 +238,7 @@ end:
 
 void prepare_single_stmt(sqlite3 *db, sqlite3_stmt **stmt, VALUE sql) {
   prepare_stmt_ctx ctx = {db, stmt, RSTRING_PTR(sql), RSTRING_LEN(sql), 0};
-  rb_thread_call_without_gvl(prepare_single_stmt_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
+  gvl_call(GVL_RELEASE, prepare_single_stmt_impl, (void *)&ctx);
   RB_GC_GUARD(sql);
 
   switch (ctx.rc) {
@@ -227,15 +260,26 @@ struct step_ctx {
   int rc;
 };
 
-void *stmt_iterate_without_gvl(void *ptr) {
+void *stmt_iterate_step(void *ptr) {
   struct step_ctx *ctx = (struct step_ctx *)ptr;
   ctx->rc = sqlite3_step(ctx->stmt);
   return NULL;
 }
 
+inline enum gvl_mode stepwise_gvl_mode(query_ctx *ctx) {
+  // a negative or zero threshold means the GVL is always held during iteration.
+  if (ctx->gvl_release_threshold <= 0) return GVL_HOLD;
+  
+  if (!sqlite3_stmt_busy(ctx->stmt)) return GVL_RELEASE;
+
+  // if positive, the GVL is normally held, and release every <threshold> steps.
+  return (ctx->step_count % ctx->gvl_release_threshold) ? GVL_HOLD : GVL_RELEASE;
+}
+
 inline int stmt_iterate(query_ctx *ctx) {
   struct step_ctx step_ctx = {ctx->stmt, 0};
-  rb_thread_call_without_gvl(stmt_iterate_without_gvl, (void *)&step_ctx, RUBY_UBF_IO, 0);
+  ctx->step_count += 1;
+  gvl_call(stepwise_gvl_mode(ctx), stmt_iterate_step, (void *)&step_ctx);
   switch (step_ctx.rc) {
     case SQLITE_ROW:
       return 1;